1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * hosting IBM Z kernel virtual machines (s390x)
4 *
5 * Copyright IBM Corp. 2008, 2020
6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Christian Ehrhardt <ehrhardt@de.ibm.com>
10 * Jason J. Herne <jjherne@us.ibm.com>
11 */
12
13 #define pr_fmt(fmt) "kvm-s390: " fmt
14
15 #include <linux/compiler.h>
16 #include <linux/entry-virt.h>
17 #include <linux/export.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/cpufeature.h>
28 #include <linux/random.h>
29 #include <linux/slab.h>
30 #include <linux/timer.h>
31 #include <linux/vmalloc.h>
32 #include <linux/bitmap.h>
33 #include <linux/sched/signal.h>
34 #include <linux/string.h>
35 #include <linux/pgtable.h>
36 #include <linux/mmu_notifier.h>
37
38 #include <asm/access-regs.h>
39 #include <asm/asm-offsets.h>
40 #include <asm/lowcore.h>
41 #include <asm/machine.h>
42 #include <asm/stp.h>
43 #include <asm/gmap_helpers.h>
44 #include <asm/nmi.h>
45 #include <asm/isc.h>
46 #include <asm/sclp.h>
47 #include <asm/cpacf.h>
48 #include <asm/timex.h>
49 #include <asm/asm.h>
50 #include <asm/fpu.h>
51 #include <asm/ap.h>
52 #include <asm/uv.h>
53 #include "s390.h"
54 #include "gaccess.h"
55 #include "gmap.h"
56 #include "faultin.h"
57 #include "pci.h"
58 #include "kvm_mmu.h"
59
60 #define CREATE_TRACE_POINTS
61 #include "trace.h"
62 #include "trace-s390.h"
63
64 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
65 #define LOCAL_IRQS 32
66 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
67 (KVM_MAX_VCPUS + LOCAL_IRQS))
68
69 const struct kvm_stats_desc kvm_vm_stats_desc[] = {
70 KVM_GENERIC_VM_STATS(),
71 STATS_DESC_COUNTER(VM, inject_io),
72 STATS_DESC_COUNTER(VM, io_390_adapter_map),
73 STATS_DESC_COUNTER(VM, io_390_adapter_unmap),
74 STATS_DESC_COUNTER(VM, io_390_inatomic),
75 STATS_DESC_COUNTER(VM, io_flic_inject_airq),
76 STATS_DESC_COUNTER(VM, io_set_adapter_int),
77 STATS_DESC_COUNTER(VM, io_390_inatomic_no_inject),
78 STATS_DESC_COUNTER(VM, inject_float_mchk),
79 STATS_DESC_COUNTER(VM, inject_pfault_done),
80 STATS_DESC_COUNTER(VM, inject_service_signal),
81 STATS_DESC_COUNTER(VM, inject_virtio),
82 STATS_DESC_COUNTER(VM, aen_forward),
83 STATS_DESC_COUNTER(VM, gmap_shadow_reuse),
84 STATS_DESC_COUNTER(VM, gmap_shadow_create),
85 STATS_DESC_COUNTER(VM, gmap_shadow_r1_entry),
86 STATS_DESC_COUNTER(VM, gmap_shadow_r2_entry),
87 STATS_DESC_COUNTER(VM, gmap_shadow_r3_entry),
88 STATS_DESC_COUNTER(VM, gmap_shadow_sg_entry),
89 STATS_DESC_COUNTER(VM, gmap_shadow_pg_entry),
90 };
91
92 const struct kvm_stats_header kvm_vm_stats_header = {
93 .name_size = KVM_STATS_NAME_SIZE,
94 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
95 .id_offset = sizeof(struct kvm_stats_header),
96 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
97 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
98 sizeof(kvm_vm_stats_desc),
99 };
100
101 const struct kvm_stats_desc kvm_vcpu_stats_desc[] = {
102 KVM_GENERIC_VCPU_STATS(),
103 STATS_DESC_COUNTER(VCPU, exit_userspace),
104 STATS_DESC_COUNTER(VCPU, exit_null),
105 STATS_DESC_COUNTER(VCPU, exit_external_request),
106 STATS_DESC_COUNTER(VCPU, exit_io_request),
107 STATS_DESC_COUNTER(VCPU, exit_external_interrupt),
108 STATS_DESC_COUNTER(VCPU, exit_stop_request),
109 STATS_DESC_COUNTER(VCPU, exit_validity),
110 STATS_DESC_COUNTER(VCPU, exit_instruction),
111 STATS_DESC_COUNTER(VCPU, exit_pei),
112 STATS_DESC_COUNTER(VCPU, halt_no_poll_steal),
113 STATS_DESC_COUNTER(VCPU, instruction_lctl),
114 STATS_DESC_COUNTER(VCPU, instruction_lctlg),
115 STATS_DESC_COUNTER(VCPU, instruction_stctl),
116 STATS_DESC_COUNTER(VCPU, instruction_stctg),
117 STATS_DESC_COUNTER(VCPU, exit_program_interruption),
118 STATS_DESC_COUNTER(VCPU, exit_instr_and_program),
119 STATS_DESC_COUNTER(VCPU, exit_operation_exception),
120 STATS_DESC_COUNTER(VCPU, deliver_ckc),
121 STATS_DESC_COUNTER(VCPU, deliver_cputm),
122 STATS_DESC_COUNTER(VCPU, deliver_external_call),
123 STATS_DESC_COUNTER(VCPU, deliver_emergency_signal),
124 STATS_DESC_COUNTER(VCPU, deliver_service_signal),
125 STATS_DESC_COUNTER(VCPU, deliver_virtio),
126 STATS_DESC_COUNTER(VCPU, deliver_stop_signal),
127 STATS_DESC_COUNTER(VCPU, deliver_prefix_signal),
128 STATS_DESC_COUNTER(VCPU, deliver_restart_signal),
129 STATS_DESC_COUNTER(VCPU, deliver_program),
130 STATS_DESC_COUNTER(VCPU, deliver_io),
131 STATS_DESC_COUNTER(VCPU, deliver_machine_check),
132 STATS_DESC_COUNTER(VCPU, exit_wait_state),
133 STATS_DESC_COUNTER(VCPU, inject_ckc),
134 STATS_DESC_COUNTER(VCPU, inject_cputm),
135 STATS_DESC_COUNTER(VCPU, inject_external_call),
136 STATS_DESC_COUNTER(VCPU, inject_emergency_signal),
137 STATS_DESC_COUNTER(VCPU, inject_mchk),
138 STATS_DESC_COUNTER(VCPU, inject_pfault_init),
139 STATS_DESC_COUNTER(VCPU, inject_program),
140 STATS_DESC_COUNTER(VCPU, inject_restart),
141 STATS_DESC_COUNTER(VCPU, inject_set_prefix),
142 STATS_DESC_COUNTER(VCPU, inject_stop_signal),
143 STATS_DESC_COUNTER(VCPU, instruction_epsw),
144 STATS_DESC_COUNTER(VCPU, instruction_gs),
145 STATS_DESC_COUNTER(VCPU, instruction_io_other),
146 STATS_DESC_COUNTER(VCPU, instruction_lpsw),
147 STATS_DESC_COUNTER(VCPU, instruction_lpswe),
148 STATS_DESC_COUNTER(VCPU, instruction_lpswey),
149 STATS_DESC_COUNTER(VCPU, instruction_pfmf),
150 STATS_DESC_COUNTER(VCPU, instruction_ptff),
151 STATS_DESC_COUNTER(VCPU, instruction_sck),
152 STATS_DESC_COUNTER(VCPU, instruction_sckpf),
153 STATS_DESC_COUNTER(VCPU, instruction_stidp),
154 STATS_DESC_COUNTER(VCPU, instruction_spx),
155 STATS_DESC_COUNTER(VCPU, instruction_stpx),
156 STATS_DESC_COUNTER(VCPU, instruction_stap),
157 STATS_DESC_COUNTER(VCPU, instruction_iske),
158 STATS_DESC_COUNTER(VCPU, instruction_ri),
159 STATS_DESC_COUNTER(VCPU, instruction_rrbe),
160 STATS_DESC_COUNTER(VCPU, instruction_sske),
161 STATS_DESC_COUNTER(VCPU, instruction_ipte_interlock),
162 STATS_DESC_COUNTER(VCPU, instruction_stsi),
163 STATS_DESC_COUNTER(VCPU, instruction_stfl),
164 STATS_DESC_COUNTER(VCPU, instruction_tb),
165 STATS_DESC_COUNTER(VCPU, instruction_tpi),
166 STATS_DESC_COUNTER(VCPU, instruction_tprot),
167 STATS_DESC_COUNTER(VCPU, instruction_tsch),
168 STATS_DESC_COUNTER(VCPU, instruction_sie),
169 STATS_DESC_COUNTER(VCPU, instruction_essa),
170 STATS_DESC_COUNTER(VCPU, instruction_sthyi),
171 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense),
172 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense_running),
173 STATS_DESC_COUNTER(VCPU, instruction_sigp_external_call),
174 STATS_DESC_COUNTER(VCPU, instruction_sigp_emergency),
175 STATS_DESC_COUNTER(VCPU, instruction_sigp_cond_emergency),
176 STATS_DESC_COUNTER(VCPU, instruction_sigp_start),
177 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop),
178 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop_store_status),
179 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_status),
180 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_adtl_status),
181 STATS_DESC_COUNTER(VCPU, instruction_sigp_arch),
182 STATS_DESC_COUNTER(VCPU, instruction_sigp_prefix),
183 STATS_DESC_COUNTER(VCPU, instruction_sigp_restart),
184 STATS_DESC_COUNTER(VCPU, instruction_sigp_init_cpu_reset),
185 STATS_DESC_COUNTER(VCPU, instruction_sigp_cpu_reset),
186 STATS_DESC_COUNTER(VCPU, instruction_sigp_unknown),
187 STATS_DESC_COUNTER(VCPU, instruction_diagnose_10),
188 STATS_DESC_COUNTER(VCPU, instruction_diagnose_44),
189 STATS_DESC_COUNTER(VCPU, instruction_diagnose_9c),
190 STATS_DESC_COUNTER(VCPU, diag_9c_ignored),
191 STATS_DESC_COUNTER(VCPU, diag_9c_forward),
192 STATS_DESC_COUNTER(VCPU, instruction_diagnose_258),
193 STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),
194 STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),
195 STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),
196 STATS_DESC_COUNTER(VCPU, pfault_sync),
197 STATS_DESC_COUNTER(VCPU, signal_exits)
198 };
199
200 const struct kvm_stats_header kvm_vcpu_stats_header = {
201 .name_size = KVM_STATS_NAME_SIZE,
202 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
203 .id_offset = sizeof(struct kvm_stats_header),
204 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
205 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
206 sizeof(kvm_vcpu_stats_desc),
207 };
208
209 /* allow nested virtualization in KVM (if enabled by user space) */
210 static int nested;
211 module_param(nested, int, S_IRUGO);
212 MODULE_PARM_DESC(nested, "Nested virtualization support");
213
214 /* allow 1m huge page guest backing */
215 static int hpage;
216 module_param(hpage, int, 0444);
217 MODULE_PARM_DESC(hpage, "1m huge page backing support");
218
219 /* allow 2g huge page guest backing */
220 static int hpage_2g;
221 module_param(hpage_2g, int, 0444);
222 MODULE_PARM_DESC(hpage_2g, "2g huge page backing support");
223
224 /* maximum percentage of steal time for polling. >100 is treated like 100 */
225 static u8 halt_poll_max_steal = 10;
226 module_param(halt_poll_max_steal, byte, 0644);
227 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
228
229 /* if set to true, the GISA will be initialized and used if available */
230 static bool use_gisa = true;
231 module_param(use_gisa, bool, 0644);
232 MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
233
234 /* maximum diag9c forwarding per second */
235 unsigned int diag9c_forwarding_hz;
236 module_param(diag9c_forwarding_hz, uint, 0644);
237 MODULE_PARM_DESC(diag9c_forwarding_hz, "Maximum diag9c forwarding per second, 0 to turn off");
238
239 /*
240 * allow asynchronous deinit for protected guests; enable by default since
241 * the feature is opt-in anyway
242 */
243 static int async_destroy = 1;
244 module_param(async_destroy, int, 0444);
245 MODULE_PARM_DESC(async_destroy, "Asynchronous destroy for protected guests");
246
247 #define HMFAI_DWORDS 16
248 /*
249 * Base feature mask that defines default mask for facilities. Consists of the
250 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
251 */
252 static unsigned long kvm_s390_fac_base[HMFAI_DWORDS] = { FACILITIES_KVM };
253 static_assert(ARRAY_SIZE(((long[]){ FACILITIES_KVM })) <= HMFAI_DWORDS);
254 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_MASK_SIZE_U64);
255 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_LIST_SIZE_U64);
256 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= ARRAY_SIZE(stfle_fac_list));
257
258 /*
259 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
260 * and defines the facilities that can be enabled via a cpu model.
261 */
262 static const unsigned long kvm_s390_fac_ext[] = { FACILITIES_KVM_CPUMODEL };
263 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_MASK_SIZE_U64);
264 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_LIST_SIZE_U64);
265 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= ARRAY_SIZE(stfle_fac_list));
266
267 /* available cpu features supported by kvm */
268 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
269 /* available subfunctions indicated via query / "test bit" */
270 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
271
272 debug_info_t *kvm_s390_dbf;
273 debug_info_t *kvm_s390_dbf_uv;
274
275 /* Section: not file related */
276 /* forward declarations */
kvm_clock_sync_scb(struct kvm_s390_sie_block * scb,u64 delta)277 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
278 {
279 u8 delta_idx = 0;
280
281 /*
282 * The TOD jumps by delta, we have to compensate this by adding
283 * -delta to the epoch.
284 */
285 delta = -delta;
286
287 /* sign-extension - we're adding to signed values below */
288 if ((s64)delta < 0)
289 delta_idx = -1;
290
291 scb->epoch += delta;
292 if (scb->ecd & ECD_MEF) {
293 scb->epdx += delta_idx;
294 if (scb->epoch < delta)
295 scb->epdx += 1;
296 }
297 }
298
299 /*
300 * This callback is executed during stop_machine(). All CPUs are therefore
301 * temporarily stopped. In order not to change guest behavior, we have to
302 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
303 * so a CPU won't be stopped while calculating with the epoch.
304 */
kvm_clock_sync(struct notifier_block * notifier,unsigned long val,void * v)305 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
306 void *v)
307 {
308 struct kvm *kvm;
309 struct kvm_vcpu *vcpu;
310 unsigned long i;
311 unsigned long long *delta = v;
312
313 list_for_each_entry(kvm, &vm_list, vm_list) {
314 kvm_for_each_vcpu(i, vcpu, kvm) {
315 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
316 if (i == 0) {
317 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
318 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
319 }
320 if (vcpu->arch.cputm_enabled)
321 vcpu->arch.cputm_start += *delta;
322 if (vcpu->arch.vsie_block)
323 kvm_clock_sync_scb(vcpu->arch.vsie_block,
324 *delta);
325 }
326 }
327 return NOTIFY_OK;
328 }
329
330 static struct notifier_block kvm_clock_notifier = {
331 .notifier_call = kvm_clock_sync,
332 };
333
allow_cpu_feat(unsigned long nr)334 static void allow_cpu_feat(unsigned long nr)
335 {
336 set_bit_inv(nr, kvm_s390_available_cpu_feat);
337 }
338
plo_test_bit(unsigned char nr)339 static inline int plo_test_bit(unsigned char nr)
340 {
341 unsigned long function = (unsigned long)nr | 0x100;
342 int cc;
343
344 asm volatile(
345 " lgr 0,%[function]\n"
346 /* Parameter registers are ignored for "test bit" */
347 " plo 0,0,0,0(0)\n"
348 CC_IPM(cc)
349 : CC_OUT(cc, cc)
350 : [function] "d" (function)
351 : CC_CLOBBER_LIST("0"));
352 return CC_TRANSFORM(cc) == 0;
353 }
354
pfcr_query(u8 (* query)[16])355 static __always_inline void pfcr_query(u8 (*query)[16])
356 {
357 asm volatile(
358 " lghi 0,0\n"
359 " .insn rsy,0xeb0000000016,0,0,%[query]"
360 : [query] "=QS" (*query)
361 :
362 : "cc", "0");
363 }
364
__sortl_query(u8 (* query)[32])365 static __always_inline void __sortl_query(u8 (*query)[32])
366 {
367 asm volatile(
368 " lghi 0,0\n"
369 " la 1,%[query]\n"
370 /* Parameter registers are ignored */
371 " .insn rre,0xb9380000,2,4"
372 : [query] "=R" (*query)
373 :
374 : "cc", "0", "1");
375 }
376
__dfltcc_query(u8 (* query)[32])377 static __always_inline void __dfltcc_query(u8 (*query)[32])
378 {
379 asm volatile(
380 " lghi 0,0\n"
381 " la 1,%[query]\n"
382 /* Parameter registers are ignored */
383 " .insn rrf,0xb9390000,2,4,6,0"
384 : [query] "=R" (*query)
385 :
386 : "cc", "0", "1");
387 }
388
kvm_s390_cpu_feat_init(void)389 static void __init kvm_s390_cpu_feat_init(void)
390 {
391 int i;
392
393 for (i = 0; i < 256; ++i) {
394 if (plo_test_bit(i))
395 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
396 }
397
398 if (test_facility(28)) /* TOD-clock steering */
399 ptff(kvm_s390_available_subfunc.ptff,
400 sizeof(kvm_s390_available_subfunc.ptff),
401 PTFF_QAF);
402
403 if (test_facility(17)) { /* MSA */
404 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
405 kvm_s390_available_subfunc.kmac);
406 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
407 kvm_s390_available_subfunc.kmc);
408 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
409 kvm_s390_available_subfunc.km);
410 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
411 kvm_s390_available_subfunc.kimd);
412 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
413 kvm_s390_available_subfunc.klmd);
414 }
415 if (test_facility(76)) /* MSA3 */
416 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
417 kvm_s390_available_subfunc.pckmo);
418 if (test_facility(77)) { /* MSA4 */
419 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
420 kvm_s390_available_subfunc.kmctr);
421 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
422 kvm_s390_available_subfunc.kmf);
423 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
424 kvm_s390_available_subfunc.kmo);
425 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
426 kvm_s390_available_subfunc.pcc);
427 }
428 if (test_facility(57)) /* MSA5 */
429 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
430 kvm_s390_available_subfunc.ppno);
431
432 if (test_facility(146)) /* MSA8 */
433 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
434 kvm_s390_available_subfunc.kma);
435
436 if (test_facility(155)) /* MSA9 */
437 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
438 kvm_s390_available_subfunc.kdsa);
439
440 if (test_facility(150)) /* SORTL */
441 __sortl_query(&kvm_s390_available_subfunc.sortl);
442
443 if (test_facility(151)) /* DFLTCC */
444 __dfltcc_query(&kvm_s390_available_subfunc.dfltcc);
445
446 if (test_facility(201)) /* PFCR */
447 pfcr_query(&kvm_s390_available_subfunc.pfcr);
448
449 if (machine_has_esop())
450 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
451 /*
452 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
453 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
454 */
455 if (!sclp.has_sief2 || !machine_has_esop() || !sclp.has_64bscao ||
456 !test_facility(3) || !nested)
457 return;
458 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
459 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
460 if (sclp.has_siif)
461 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
462 if (sclp.has_gpere)
463 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
464 if (sclp.has_gsls)
465 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
466 if (sclp.has_ib)
467 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
468 if (sclp.has_cei)
469 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
470 if (sclp.has_ibs)
471 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
472 if (sclp.has_kss)
473 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
474 if (sclp.has_astfleie2)
475 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ASTFLEIE2);
476 /*
477 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
478 * all skey handling functions read/set the skey from the PGSTE
479 * instead of the real storage key.
480 *
481 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
482 * pages being detected as preserved although they are resident.
483 *
484 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
485 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
486 *
487 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
488 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
489 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
490 *
491 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
492 * cannot easily shadow the SCA because of the ipte lock.
493 */
494 }
495
__kvm_s390_init(void)496 static int __init __kvm_s390_init(void)
497 {
498 int rc = -ENOMEM;
499
500 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
501 if (!kvm_s390_dbf)
502 return -ENOMEM;
503
504 kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
505 if (!kvm_s390_dbf_uv)
506 goto err_kvm_uv;
507
508 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
509 debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
510 goto err_debug_view;
511
512 kvm_s390_cpu_feat_init();
513
514 /* Register floating interrupt controller interface. */
515 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
516 if (rc) {
517 pr_err("A FLIC registration call failed with rc=%d\n", rc);
518 goto err_flic;
519 }
520
521 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
522 rc = kvm_s390_pci_init();
523 if (rc) {
524 pr_err("Unable to allocate AIFT for PCI\n");
525 goto err_pci;
526 }
527 }
528
529 rc = kvm_s390_gib_init(GAL_ISC);
530 if (rc)
531 goto err_gib;
532
533 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
534 &kvm_clock_notifier);
535
536 return 0;
537
538 err_gib:
539 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
540 kvm_s390_pci_exit();
541 err_pci:
542 err_flic:
543 err_debug_view:
544 debug_unregister(kvm_s390_dbf_uv);
545 err_kvm_uv:
546 debug_unregister(kvm_s390_dbf);
547 return rc;
548 }
549
__kvm_s390_exit(void)550 static void __kvm_s390_exit(void)
551 {
552 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
553 &kvm_clock_notifier);
554
555 kvm_s390_gib_destroy();
556 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
557 kvm_s390_pci_exit();
558 debug_unregister(kvm_s390_dbf);
559 debug_unregister(kvm_s390_dbf_uv);
560 }
561
kvm_s390_keyop(struct kvm_s390_mmu_cache * mc,struct kvm * kvm,int op,unsigned long addr,union skey skey)562 static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op,
563 unsigned long addr, union skey skey)
564 {
565 gfn_t gfn = gpa_to_gfn(addr);
566 union asce asce;
567 int r;
568
569 guard(read_lock)(&kvm->mmu_lock);
570 asce = kvm->arch.gmap->asce;
571
572 switch (op) {
573 case KVM_S390_KEYOP_SSKE:
574 r = dat_cond_set_storage_key(mc, asce, gfn, skey, &skey, 0, 0, 0);
575 if (r == 0 || r == 1)
576 return skey.skey;
577 break;
578 case KVM_S390_KEYOP_ISKE:
579 r = dat_get_storage_key(asce, gfn, &skey);
580 if (!r)
581 return skey.skey;
582 break;
583 case KVM_S390_KEYOP_RRBE:
584 r = dat_reset_reference_bit(asce, gfn, &skey);
585 if (!r)
586 return skey.skey;
587 break;
588 default:
589 return -EINVAL;
590 }
591 return r > 0 ? -EFAULT : r;
592 }
593
594 /* Section: device related */
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)595 long kvm_arch_dev_ioctl(struct file *filp,
596 unsigned int ioctl, unsigned long arg)
597 {
598 if (ioctl == KVM_S390_ENABLE_SIE)
599 return 0;
600 return -EINVAL;
601 }
602
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)603 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
604 {
605 int r;
606
607 switch (ext) {
608 case KVM_CAP_S390_PSW:
609 case KVM_CAP_S390_GMAP:
610 #ifdef CONFIG_KVM_S390_UCONTROL
611 case KVM_CAP_S390_UCONTROL:
612 #endif
613 case KVM_CAP_ASYNC_PF:
614 case KVM_CAP_SYNC_REGS:
615 case KVM_CAP_ONE_REG:
616 case KVM_CAP_ENABLE_CAP:
617 case KVM_CAP_S390_CSS_SUPPORT:
618 case KVM_CAP_IOEVENTFD:
619 case KVM_CAP_S390_IRQCHIP:
620 case KVM_CAP_VM_ATTRIBUTES:
621 case KVM_CAP_MP_STATE:
622 case KVM_CAP_IMMEDIATE_EXIT:
623 case KVM_CAP_S390_INJECT_IRQ:
624 case KVM_CAP_S390_USER_SIGP:
625 case KVM_CAP_S390_USER_STSI:
626 case KVM_CAP_S390_SKEYS:
627 case KVM_CAP_S390_IRQ_STATE:
628 case KVM_CAP_S390_USER_INSTR0:
629 case KVM_CAP_S390_CMMA_MIGRATION:
630 case KVM_CAP_S390_AIS:
631 case KVM_CAP_S390_AIS_MIGRATION:
632 case KVM_CAP_S390_VCPU_RESETS:
633 case KVM_CAP_SET_GUEST_DEBUG:
634 case KVM_CAP_S390_DIAG318:
635 case KVM_CAP_IRQFD_RESAMPLE:
636 case KVM_CAP_S390_USER_OPEREXEC:
637 case KVM_CAP_S390_KEYOP:
638 case KVM_CAP_S390_VSIE_ESAMODE:
639 case KVM_CAP_PRE_FAULT_MEMORY:
640 r = 1;
641 break;
642 case KVM_CAP_SET_GUEST_DEBUG2:
643 r = KVM_GUESTDBG_VALID_MASK;
644 break;
645 case KVM_CAP_S390_HPAGE_1M:
646 r = 0;
647 if (hpage && !(kvm && kvm_is_ucontrol(kvm)))
648 r = 1;
649 break;
650 case KVM_CAP_S390_HPAGE_2G:
651 r = 0;
652 if (hpage_2g && !(kvm && kvm_is_ucontrol(kvm)))
653 r = 1;
654 break;
655 case KVM_CAP_S390_MEM_OP:
656 r = MEM_OP_MAX_SIZE;
657 break;
658 case KVM_CAP_S390_MEM_OP_EXTENSION:
659 /*
660 * Flag bits indicating which extensions are supported.
661 * If r > 0, the base extension must also be supported/indicated,
662 * in order to maintain backwards compatibility.
663 */
664 r = KVM_S390_MEMOP_EXTENSION_CAP_BASE |
665 KVM_S390_MEMOP_EXTENSION_CAP_CMPXCHG;
666 break;
667 case KVM_CAP_NR_VCPUS:
668 case KVM_CAP_MAX_VCPUS:
669 case KVM_CAP_MAX_VCPU_ID:
670 /*
671 * Return the same value for KVM_CAP_MAX_VCPUS and
672 * KVM_CAP_MAX_VCPU_ID to conform with the KVM API.
673 */
674 r = KVM_S390_ESCA_CPU_SLOTS;
675 if (!kvm_s390_use_sca_entries())
676 r = KVM_MAX_VCPUS;
677 if (ext == KVM_CAP_NR_VCPUS)
678 r = min_t(unsigned int, num_online_cpus(), r);
679 break;
680 case KVM_CAP_S390_COW:
681 r = machine_has_esop();
682 break;
683 case KVM_CAP_S390_VECTOR_REGISTERS:
684 r = test_facility(129);
685 break;
686 case KVM_CAP_S390_RI:
687 r = test_facility(64);
688 break;
689 case KVM_CAP_S390_GS:
690 r = test_facility(133);
691 break;
692 case KVM_CAP_S390_BPB:
693 r = test_facility(82);
694 break;
695 case KVM_CAP_S390_PROTECTED_ASYNC_DISABLE:
696 r = async_destroy && is_prot_virt_host();
697 break;
698 case KVM_CAP_S390_PROTECTED:
699 r = is_prot_virt_host();
700 break;
701 case KVM_CAP_S390_PROTECTED_DUMP: {
702 u64 pv_cmds_dump[] = {
703 BIT_UVC_CMD_DUMP_INIT,
704 BIT_UVC_CMD_DUMP_CONFIG_STOR_STATE,
705 BIT_UVC_CMD_DUMP_CPU,
706 BIT_UVC_CMD_DUMP_COMPLETE,
707 };
708 int i;
709
710 r = is_prot_virt_host();
711
712 for (i = 0; i < ARRAY_SIZE(pv_cmds_dump); i++) {
713 if (!test_bit_inv(pv_cmds_dump[i],
714 (unsigned long *)&uv_info.inst_calls_list)) {
715 r = 0;
716 break;
717 }
718 }
719 break;
720 }
721 case KVM_CAP_S390_ZPCI_OP:
722 r = kvm_s390_pci_interp_allowed();
723 break;
724 case KVM_CAP_S390_CPU_TOPOLOGY:
725 r = test_facility(11);
726 break;
727 default:
728 r = 0;
729 }
730 return r;
731 }
732
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)733 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
734 {
735 gfn_t last_gfn = memslot->base_gfn + memslot->npages;
736
737 scoped_guard(read_lock, &kvm->mmu_lock)
738 gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn);
739 }
740
741 /* Section: vm related */
742 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
743
744 /*
745 * Get (and clear) the dirty memory log for a memory slot.
746 */
kvm_vm_ioctl_get_dirty_log(struct kvm * kvm,struct kvm_dirty_log * log)747 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
748 struct kvm_dirty_log *log)
749 {
750 return s390_kvm_mmu_get_dirty_log(kvm, log);
751 }
752
icpt_operexc_on_all_vcpus(struct kvm * kvm)753 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
754 {
755 unsigned long i;
756 struct kvm_vcpu *vcpu;
757
758 kvm_for_each_vcpu(i, vcpu, kvm) {
759 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
760 }
761 }
762
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)763 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
764 {
765 int r;
766
767 if (cap->flags)
768 return -EINVAL;
769
770 switch (cap->cap) {
771 case KVM_CAP_S390_IRQCHIP:
772 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
773 kvm->arch.use_irqchip = 1;
774 r = 0;
775 break;
776 case KVM_CAP_S390_USER_SIGP:
777 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
778 kvm->arch.user_sigp = 1;
779 r = 0;
780 break;
781 case KVM_CAP_S390_VECTOR_REGISTERS:
782 mutex_lock(&kvm->lock);
783 if (kvm->created_vcpus) {
784 r = -EBUSY;
785 } else if (cpu_has_vx()) {
786 set_kvm_facility(kvm->arch.model.fac_mask, 129);
787 set_kvm_facility(kvm->arch.model.fac_list, 129);
788 if (test_facility(134)) {
789 set_kvm_facility(kvm->arch.model.fac_mask, 134);
790 set_kvm_facility(kvm->arch.model.fac_list, 134);
791 }
792 if (test_facility(135)) {
793 set_kvm_facility(kvm->arch.model.fac_mask, 135);
794 set_kvm_facility(kvm->arch.model.fac_list, 135);
795 }
796 if (test_facility(148)) {
797 set_kvm_facility(kvm->arch.model.fac_mask, 148);
798 set_kvm_facility(kvm->arch.model.fac_list, 148);
799 }
800 if (test_facility(152)) {
801 set_kvm_facility(kvm->arch.model.fac_mask, 152);
802 set_kvm_facility(kvm->arch.model.fac_list, 152);
803 }
804 if (test_facility(192)) {
805 set_kvm_facility(kvm->arch.model.fac_mask, 192);
806 set_kvm_facility(kvm->arch.model.fac_list, 192);
807 }
808 if (test_facility(198)) {
809 set_kvm_facility(kvm->arch.model.fac_mask, 198);
810 set_kvm_facility(kvm->arch.model.fac_list, 198);
811 }
812 if (test_facility(199)) {
813 set_kvm_facility(kvm->arch.model.fac_mask, 199);
814 set_kvm_facility(kvm->arch.model.fac_list, 199);
815 }
816 r = 0;
817 } else
818 r = -EINVAL;
819 mutex_unlock(&kvm->lock);
820 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
821 r ? "(not available)" : "(success)");
822 break;
823 case KVM_CAP_S390_RI:
824 r = -EINVAL;
825 mutex_lock(&kvm->lock);
826 if (kvm->created_vcpus) {
827 r = -EBUSY;
828 } else if (test_facility(64)) {
829 set_kvm_facility(kvm->arch.model.fac_mask, 64);
830 set_kvm_facility(kvm->arch.model.fac_list, 64);
831 r = 0;
832 }
833 mutex_unlock(&kvm->lock);
834 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
835 r ? "(not available)" : "(success)");
836 break;
837 case KVM_CAP_S390_AIS:
838 mutex_lock(&kvm->lock);
839 if (kvm->created_vcpus) {
840 r = -EBUSY;
841 } else {
842 set_kvm_facility(kvm->arch.model.fac_mask, 72);
843 set_kvm_facility(kvm->arch.model.fac_list, 72);
844 r = 0;
845 }
846 mutex_unlock(&kvm->lock);
847 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
848 r ? "(not available)" : "(success)");
849 break;
850 case KVM_CAP_S390_GS:
851 r = -EINVAL;
852 mutex_lock(&kvm->lock);
853 if (kvm->created_vcpus) {
854 r = -EBUSY;
855 } else if (test_facility(133)) {
856 set_kvm_facility(kvm->arch.model.fac_mask, 133);
857 set_kvm_facility(kvm->arch.model.fac_list, 133);
858 r = 0;
859 }
860 mutex_unlock(&kvm->lock);
861 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
862 r ? "(not available)" : "(success)");
863 break;
864 case KVM_CAP_S390_HPAGE_1M:
865 mutex_lock(&kvm->lock);
866 if (kvm->created_vcpus)
867 r = -EBUSY;
868 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
869 r = -EINVAL;
870 else {
871 r = 0;
872 set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
873 /*
874 * We might have to create fake 4k page
875 * tables. To avoid that the hardware works on
876 * stale PGSTEs, we emulate these instructions.
877 */
878 kvm->arch.use_skf = 0;
879 kvm->arch.use_pfmfi = 0;
880 }
881 mutex_unlock(&kvm->lock);
882 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
883 r ? "(not available)" : "(success)");
884 break;
885 case KVM_CAP_S390_HPAGE_2G:
886 mutex_lock(&kvm->lock);
887 if (kvm->created_vcpus) {
888 r = -EBUSY;
889 } else if (!hpage_2g || kvm->arch.use_cmma || kvm_is_ucontrol(kvm)) {
890 r = -EINVAL;
891 } else {
892 r = 0;
893 set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags);
894 /*
895 * We might have to create fake 4k page
896 * tables. To avoid that the hardware works on
897 * stale PGSTEs, we emulate these instructions.
898 */
899 kvm->arch.use_skf = 0;
900 kvm->arch.use_pfmfi = 0;
901 }
902 mutex_unlock(&kvm->lock);
903 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE_2G %s",
904 r ? "(not available)" : "(success)");
905 break;
906 case KVM_CAP_S390_USER_STSI:
907 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
908 kvm->arch.user_stsi = 1;
909 r = 0;
910 break;
911 case KVM_CAP_S390_USER_INSTR0:
912 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
913 kvm->arch.user_instr0 = 1;
914 icpt_operexc_on_all_vcpus(kvm);
915 r = 0;
916 break;
917 case KVM_CAP_S390_CPU_TOPOLOGY:
918 r = -EINVAL;
919 mutex_lock(&kvm->lock);
920 if (kvm->created_vcpus) {
921 r = -EBUSY;
922 } else if (test_facility(11)) {
923 set_kvm_facility(kvm->arch.model.fac_mask, 11);
924 set_kvm_facility(kvm->arch.model.fac_list, 11);
925 r = 0;
926 }
927 mutex_unlock(&kvm->lock);
928 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",
929 r ? "(not available)" : "(success)");
930 break;
931 case KVM_CAP_S390_USER_OPEREXEC:
932 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_OPEREXEC");
933 kvm->arch.user_operexec = 1;
934 icpt_operexc_on_all_vcpus(kvm);
935 r = 0;
936 break;
937 case KVM_CAP_S390_VSIE_ESAMODE:
938 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_VSIE_ESAMODE");
939 kvm->arch.allow_vsie_esamode = 1;
940 r = 0;
941 break;
942 default:
943 r = -EINVAL;
944 break;
945 }
946 return r;
947 }
948
kvm_s390_get_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)949 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
950 {
951 int ret;
952
953 switch (attr->attr) {
954 case KVM_S390_VM_MEM_LIMIT_SIZE:
955 ret = 0;
956 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
957 kvm->arch.mem_limit);
958 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
959 ret = -EFAULT;
960 break;
961 default:
962 ret = -ENXIO;
963 break;
964 }
965 return ret;
966 }
967
kvm_s390_set_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)968 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
969 {
970 int ret;
971
972 switch (attr->attr) {
973 case KVM_S390_VM_MEM_ENABLE_CMMA:
974 ret = -ENXIO;
975 if (!sclp.has_cmma)
976 break;
977
978 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
979 mutex_lock(&kvm->lock);
980 if (kvm->created_vcpus)
981 ret = -EBUSY;
982 else {
983 kvm->arch.use_cmma = 1;
984 /* Not compatible with cmma. */
985 kvm->arch.use_pfmfi = 0;
986 ret = 0;
987 }
988 mutex_unlock(&kvm->lock);
989 break;
990 case KVM_S390_VM_MEM_CLR_CMMA: {
991 gfn_t start_gfn = 0;
992
993 ret = -ENXIO;
994 if (!sclp.has_cmma)
995 break;
996 ret = -EINVAL;
997 if (!kvm->arch.use_cmma)
998 break;
999
1000 guard(mutex)(&kvm->lock);
1001 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
1002 do {
1003 scoped_guard(read_lock, &kvm->mmu_lock)
1004 start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn);
1005 } while (start_gfn);
1006 ret = 0;
1007 break;
1008 }
1009 case KVM_S390_VM_MEM_LIMIT_SIZE: {
1010 struct kvm_memslots *slots;
1011 struct kvm_memory_slot *ms;
1012 unsigned long new_limit;
1013 int bkt;
1014
1015 if (kvm_is_ucontrol(kvm))
1016 return -EINVAL;
1017
1018 if (get_user(new_limit, (u64 __user *)attr->addr))
1019 return -EFAULT;
1020
1021 guard(mutex)(&kvm->lock);
1022
1023 new_limit = ALIGN(new_limit, HPAGE_SIZE);
1024 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
1025 new_limit > kvm->arch.mem_limit)
1026 return -E2BIG;
1027
1028 if (!new_limit)
1029 return -EINVAL;
1030
1031 if (kvm->created_vcpus)
1032 return -EBUSY;
1033
1034 ret = 0;
1035 scoped_guard(mutex, &kvm->slots_lock) {
1036 slots = kvm_memslots(kvm);
1037 if (slots && !kvm_memslots_empty(slots)) {
1038 kvm_for_each_memslot(ms, bkt, slots) {
1039 if (gpa_to_gfn(new_limit) < ms->base_gfn + ms->npages) {
1040 ret = -EBUSY;
1041 break;
1042 }
1043 }
1044 }
1045 if (!ret)
1046 ret = gmap_set_limit(kvm->arch.gmap, gpa_to_gfn(new_limit));
1047 }
1048 if (ret)
1049 break;
1050 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
1051 VM_EVENT(kvm, 3, "New guest asce: 0x%p", (void *)kvm->arch.gmap->asce.val);
1052 break;
1053 }
1054 default:
1055 ret = -ENXIO;
1056 break;
1057 }
1058 return ret;
1059 }
1060
1061 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
1062
kvm_s390_vcpu_crypto_reset_all(struct kvm * kvm)1063 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
1064 {
1065 struct kvm_vcpu *vcpu;
1066 unsigned long i;
1067
1068 kvm_s390_vcpu_block_all(kvm);
1069
1070 kvm_for_each_vcpu(i, vcpu, kvm) {
1071 kvm_s390_vcpu_crypto_setup(vcpu);
1072 /* recreate the shadow crycb by leaving the VSIE handler */
1073 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1074 }
1075
1076 kvm_s390_vcpu_unblock_all(kvm);
1077 }
1078
kvm_s390_vm_set_crypto(struct kvm * kvm,struct kvm_device_attr * attr)1079 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
1080 {
1081 mutex_lock(&kvm->lock);
1082 switch (attr->attr) {
1083 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1084 if (!test_kvm_facility(kvm, 76)) {
1085 mutex_unlock(&kvm->lock);
1086 return -EINVAL;
1087 }
1088 get_random_bytes(
1089 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1090 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1091 kvm->arch.crypto.aes_kw = 1;
1092 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
1093 break;
1094 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1095 if (!test_kvm_facility(kvm, 76)) {
1096 mutex_unlock(&kvm->lock);
1097 return -EINVAL;
1098 }
1099 get_random_bytes(
1100 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1101 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1102 kvm->arch.crypto.dea_kw = 1;
1103 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
1104 break;
1105 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1106 if (!test_kvm_facility(kvm, 76)) {
1107 mutex_unlock(&kvm->lock);
1108 return -EINVAL;
1109 }
1110 kvm->arch.crypto.aes_kw = 0;
1111 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
1112 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1113 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
1114 break;
1115 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1116 if (!test_kvm_facility(kvm, 76)) {
1117 mutex_unlock(&kvm->lock);
1118 return -EINVAL;
1119 }
1120 kvm->arch.crypto.dea_kw = 0;
1121 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
1122 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1123 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
1124 break;
1125 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1126 if (!ap_instructions_available()) {
1127 mutex_unlock(&kvm->lock);
1128 return -EOPNOTSUPP;
1129 }
1130 kvm->arch.crypto.apie = 1;
1131 break;
1132 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1133 if (!ap_instructions_available()) {
1134 mutex_unlock(&kvm->lock);
1135 return -EOPNOTSUPP;
1136 }
1137 kvm->arch.crypto.apie = 0;
1138 break;
1139 default:
1140 mutex_unlock(&kvm->lock);
1141 return -ENXIO;
1142 }
1143
1144 kvm_s390_vcpu_crypto_reset_all(kvm);
1145 mutex_unlock(&kvm->lock);
1146 return 0;
1147 }
1148
kvm_s390_vcpu_pci_setup(struct kvm_vcpu * vcpu)1149 static void kvm_s390_vcpu_pci_setup(struct kvm_vcpu *vcpu)
1150 {
1151 /* Only set the ECB bits after guest requests zPCI interpretation */
1152 if (!vcpu->kvm->arch.use_zpci_interp)
1153 return;
1154
1155 vcpu->arch.sie_block->ecb2 |= ECB2_ZPCI_LSI;
1156 vcpu->arch.sie_block->ecb3 |= ECB3_AISII + ECB3_AISI;
1157 }
1158
kvm_s390_vcpu_pci_enable_interp(struct kvm * kvm)1159 void kvm_s390_vcpu_pci_enable_interp(struct kvm *kvm)
1160 {
1161 struct kvm_vcpu *vcpu;
1162 unsigned long i;
1163
1164 lockdep_assert_held(&kvm->lock);
1165
1166 if (!kvm_s390_pci_interp_allowed())
1167 return;
1168
1169 /*
1170 * If host is configured for PCI and the necessary facilities are
1171 * available, turn on interpretation for the life of this guest
1172 */
1173 kvm->arch.use_zpci_interp = 1;
1174
1175 kvm_s390_vcpu_block_all(kvm);
1176
1177 kvm_for_each_vcpu(i, vcpu, kvm) {
1178 kvm_s390_vcpu_pci_setup(vcpu);
1179 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1180 }
1181
1182 kvm_s390_vcpu_unblock_all(kvm);
1183 }
1184
kvm_s390_sync_request_broadcast(struct kvm * kvm,int req)1185 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
1186 {
1187 unsigned long cx;
1188 struct kvm_vcpu *vcpu;
1189
1190 kvm_for_each_vcpu(cx, vcpu, kvm)
1191 kvm_s390_sync_request(req, vcpu);
1192 }
1193
1194 /*
1195 * Must be called with kvm->srcu held to avoid races on memslots, and with
1196 * kvm->slots_arch_lock to avoid races with ourselves,
1197 * kvm_s390_vm_stop_migration(), and kvm_s390_get_cmma_bits().
1198 */
kvm_s390_vm_start_migration(struct kvm * kvm)1199 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1200 {
1201 struct kvm_memory_slot *ms;
1202 struct kvm_memslots *slots;
1203 int bkt;
1204
1205 /* migration mode already enabled */
1206 if (kvm->arch.migration_mode)
1207 return 0;
1208 slots = kvm_memslots(kvm);
1209 if (!slots || kvm_memslots_empty(slots))
1210 return -EINVAL;
1211
1212 if (!kvm->arch.use_cmma) {
1213 kvm->arch.migration_mode = 1;
1214 return 0;
1215 }
1216 kvm_for_each_memslot(ms, bkt, slots) {
1217 if (!ms->dirty_bitmap)
1218 return -EINVAL;
1219 }
1220 /*
1221 * Set the flag and let KVM handle ESSA manually, potentially setting
1222 * the cmma_d bit in some PGSTEs and increasing cmma_dirty_pages.
1223 * At this point cmma_dirty_pages is still 0, and all existing PGSTEs
1224 * have their cmma_d bit set to 0.
1225 * Any newly allocated page table has its entries marked as cmma-clean,
1226 * which is fine because the CMMA values are not dirty.
1227 */
1228 WRITE_ONCE(kvm->arch.migration_mode, 1);
1229 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1230 /*
1231 * Mark all PGSTEs as cmma-dirty, increasing cmma_dirty_pages as needed,
1232 * but without double-counting pages that have become dirty on their own
1233 * in the meantime.
1234 * At this point some pages might have become dirty on their own already
1235 * and cmma_dirty_pages might therefore be non-zero.
1236 */
1237 gmap_set_cmma_all_dirty(kvm->arch.gmap);
1238 return 0;
1239 }
1240
1241 /*
1242 * Must be called with kvm->slots_arch_lock to avoid races with ourselves,
1243 * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits().
1244 */
kvm_s390_vm_stop_migration(struct kvm * kvm)1245 int kvm_s390_vm_stop_migration(struct kvm *kvm)
1246 {
1247 /* migration mode already disabled */
1248 if (!kvm->arch.migration_mode)
1249 return 0;
1250 /*
1251 * Unset the flag and propagate to all vCPUs. From now on the cmma_d
1252 * bit will not be touched on any PGSTE.
1253 * At this point cmma_dirty_pages is possibly non-zero, and thus some
1254 * PGSTEs might have cmma_d set.
1255 */
1256 WRITE_ONCE(kvm->arch.migration_mode, 0);
1257 if (!kvm->arch.use_cmma)
1258 return 0;
1259
1260 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1261 /* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */
1262 gmap_set_cmma_all_clean(kvm->arch.gmap);
1263 atomic64_set(&kvm->arch.cmma_dirty_pages, 0);
1264 /*
1265 * At this point the system has the expected state: migration_mode is 0,
1266 * cmma_dirty_pages is 0, and all existing PGSTEs have their cmma_d bit
1267 * set to 0.
1268 */
1269 return 0;
1270 }
1271
kvm_s390_vm_set_migration(struct kvm * kvm,struct kvm_device_attr * attr)1272 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1273 struct kvm_device_attr *attr)
1274 {
1275 int res = -ENXIO;
1276
1277 guard(srcu)(&kvm->srcu);
1278 guard(mutex)(&kvm->slots_arch_lock);
1279
1280 switch (attr->attr) {
1281 case KVM_S390_VM_MIGRATION_START:
1282 res = kvm_s390_vm_start_migration(kvm);
1283 break;
1284 case KVM_S390_VM_MIGRATION_STOP:
1285 res = kvm_s390_vm_stop_migration(kvm);
1286 break;
1287 default:
1288 break;
1289 }
1290
1291 return res;
1292 }
1293
kvm_s390_vm_get_migration(struct kvm * kvm,struct kvm_device_attr * attr)1294 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1295 struct kvm_device_attr *attr)
1296 {
1297 u64 mig = kvm->arch.migration_mode;
1298
1299 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1300 return -ENXIO;
1301
1302 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1303 return -EFAULT;
1304 return 0;
1305 }
1306
1307 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod);
1308
kvm_s390_set_tod_ext(struct kvm * kvm,struct kvm_device_attr * attr)1309 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1310 {
1311 struct kvm_s390_vm_tod_clock gtod;
1312
1313 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
1314 return -EFAULT;
1315
1316 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1317 return -EINVAL;
1318 __kvm_s390_set_tod_clock(kvm, >od);
1319
1320 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1321 gtod.epoch_idx, gtod.tod);
1322
1323 return 0;
1324 }
1325
kvm_s390_set_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)1326 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1327 {
1328 u8 gtod_high;
1329
1330 if (copy_from_user(>od_high, (void __user *)attr->addr,
1331 sizeof(gtod_high)))
1332 return -EFAULT;
1333
1334 if (gtod_high != 0)
1335 return -EINVAL;
1336 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1337
1338 return 0;
1339 }
1340
kvm_s390_set_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)1341 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1342 {
1343 struct kvm_s390_vm_tod_clock gtod = { 0 };
1344
1345 if (copy_from_user(>od.tod, (void __user *)attr->addr,
1346 sizeof(gtod.tod)))
1347 return -EFAULT;
1348
1349 __kvm_s390_set_tod_clock(kvm, >od);
1350 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1351 return 0;
1352 }
1353
kvm_s390_set_tod(struct kvm * kvm,struct kvm_device_attr * attr)1354 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1355 {
1356 int ret;
1357
1358 if (attr->flags)
1359 return -EINVAL;
1360
1361 mutex_lock(&kvm->lock);
1362 /*
1363 * For protected guests, the TOD is managed by the ultravisor, so trying
1364 * to change it will never bring the expected results.
1365 */
1366 if (kvm_s390_pv_is_protected(kvm)) {
1367 ret = -EOPNOTSUPP;
1368 goto out_unlock;
1369 }
1370
1371 switch (attr->attr) {
1372 case KVM_S390_VM_TOD_EXT:
1373 ret = kvm_s390_set_tod_ext(kvm, attr);
1374 break;
1375 case KVM_S390_VM_TOD_HIGH:
1376 ret = kvm_s390_set_tod_high(kvm, attr);
1377 break;
1378 case KVM_S390_VM_TOD_LOW:
1379 ret = kvm_s390_set_tod_low(kvm, attr);
1380 break;
1381 default:
1382 ret = -ENXIO;
1383 break;
1384 }
1385
1386 out_unlock:
1387 mutex_unlock(&kvm->lock);
1388 return ret;
1389 }
1390
kvm_s390_get_tod_clock(struct kvm * kvm,struct kvm_s390_vm_tod_clock * gtod)1391 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1392 struct kvm_s390_vm_tod_clock *gtod)
1393 {
1394 union tod_clock clk;
1395
1396 preempt_disable();
1397
1398 store_tod_clock_ext(&clk);
1399
1400 gtod->tod = clk.tod + kvm->arch.epoch;
1401 gtod->epoch_idx = 0;
1402 if (test_kvm_facility(kvm, 139)) {
1403 gtod->epoch_idx = clk.ei + kvm->arch.epdx;
1404 if (gtod->tod < clk.tod)
1405 gtod->epoch_idx += 1;
1406 }
1407
1408 preempt_enable();
1409 }
1410
kvm_s390_get_tod_ext(struct kvm * kvm,struct kvm_device_attr * attr)1411 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1412 {
1413 struct kvm_s390_vm_tod_clock gtod;
1414
1415 memset(>od, 0, sizeof(gtod));
1416 kvm_s390_get_tod_clock(kvm, >od);
1417 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1418 return -EFAULT;
1419
1420 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1421 gtod.epoch_idx, gtod.tod);
1422 return 0;
1423 }
1424
kvm_s390_get_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)1425 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1426 {
1427 u8 gtod_high = 0;
1428
1429 if (copy_to_user((void __user *)attr->addr, >od_high,
1430 sizeof(gtod_high)))
1431 return -EFAULT;
1432 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1433
1434 return 0;
1435 }
1436
kvm_s390_get_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)1437 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1438 {
1439 u64 gtod;
1440
1441 gtod = kvm_s390_get_tod_clock_fast(kvm);
1442 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1443 return -EFAULT;
1444 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1445
1446 return 0;
1447 }
1448
kvm_s390_get_tod(struct kvm * kvm,struct kvm_device_attr * attr)1449 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1450 {
1451 int ret;
1452
1453 if (attr->flags)
1454 return -EINVAL;
1455
1456 switch (attr->attr) {
1457 case KVM_S390_VM_TOD_EXT:
1458 ret = kvm_s390_get_tod_ext(kvm, attr);
1459 break;
1460 case KVM_S390_VM_TOD_HIGH:
1461 ret = kvm_s390_get_tod_high(kvm, attr);
1462 break;
1463 case KVM_S390_VM_TOD_LOW:
1464 ret = kvm_s390_get_tod_low(kvm, attr);
1465 break;
1466 default:
1467 ret = -ENXIO;
1468 break;
1469 }
1470 return ret;
1471 }
1472
kvm_s390_set_processor(struct kvm * kvm,struct kvm_device_attr * attr)1473 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1474 {
1475 struct kvm_s390_vm_cpu_processor *proc;
1476 u16 lowest_ibc, unblocked_ibc;
1477 int ret = 0;
1478
1479 mutex_lock(&kvm->lock);
1480 if (kvm->created_vcpus) {
1481 ret = -EBUSY;
1482 goto out;
1483 }
1484 proc = kzalloc_obj(*proc, GFP_KERNEL_ACCOUNT);
1485 if (!proc) {
1486 ret = -ENOMEM;
1487 goto out;
1488 }
1489 if (!copy_from_user(proc, (void __user *)attr->addr,
1490 sizeof(*proc))) {
1491 kvm->arch.model.cpuid = proc->cpuid;
1492 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1493 unblocked_ibc = sclp.ibc & 0xfff;
1494 if (lowest_ibc && proc->ibc) {
1495 if (proc->ibc > unblocked_ibc)
1496 kvm->arch.model.ibc = unblocked_ibc;
1497 else if (proc->ibc < lowest_ibc)
1498 kvm->arch.model.ibc = lowest_ibc;
1499 else
1500 kvm->arch.model.ibc = proc->ibc;
1501 }
1502 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1503 S390_ARCH_FAC_LIST_SIZE_BYTE);
1504 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1505 kvm->arch.model.ibc,
1506 kvm->arch.model.cpuid);
1507 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1508 kvm->arch.model.fac_list[0],
1509 kvm->arch.model.fac_list[1],
1510 kvm->arch.model.fac_list[2]);
1511 } else
1512 ret = -EFAULT;
1513 kfree(proc);
1514 out:
1515 mutex_unlock(&kvm->lock);
1516 return ret;
1517 }
1518
kvm_s390_set_processor_feat(struct kvm * kvm,struct kvm_device_attr * attr)1519 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1520 struct kvm_device_attr *attr)
1521 {
1522 struct kvm_s390_vm_cpu_feat data;
1523
1524 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1525 return -EFAULT;
1526 if (!bitmap_subset((unsigned long *) data.feat,
1527 kvm_s390_available_cpu_feat,
1528 KVM_S390_VM_CPU_FEAT_NR_BITS))
1529 return -EINVAL;
1530
1531 mutex_lock(&kvm->lock);
1532 if (kvm->created_vcpus) {
1533 mutex_unlock(&kvm->lock);
1534 return -EBUSY;
1535 }
1536 bitmap_from_arr64(kvm->arch.cpu_feat, data.feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1537 mutex_unlock(&kvm->lock);
1538 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1539 data.feat[0],
1540 data.feat[1],
1541 data.feat[2]);
1542 return 0;
1543 }
1544
kvm_s390_set_processor_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1545 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1546 struct kvm_device_attr *attr)
1547 {
1548 mutex_lock(&kvm->lock);
1549 if (kvm->created_vcpus) {
1550 mutex_unlock(&kvm->lock);
1551 return -EBUSY;
1552 }
1553
1554 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1555 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1556 mutex_unlock(&kvm->lock);
1557 return -EFAULT;
1558 }
1559 mutex_unlock(&kvm->lock);
1560
1561 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1562 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1563 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1564 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1565 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1566 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1567 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1568 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1569 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1570 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1571 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1572 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1573 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1574 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1575 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1576 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1577 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1578 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1579 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1580 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1581 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1582 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1583 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1584 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1585 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1586 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1587 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1588 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1589 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1590 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1591 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1592 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1593 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1594 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1595 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1596 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1597 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1598 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1599 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1600 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1601 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1602 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1603 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1604 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1605 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1606 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1607 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1608 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1609 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1610 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1611 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1612 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1613 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1614 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1615 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1616 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1617 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1618 VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1619 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1620 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1621
1622 return 0;
1623 }
1624
1625 #define KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK \
1626 ( \
1627 ((struct kvm_s390_vm_cpu_uv_feat){ \
1628 .ap = 1, \
1629 .ap_intr = 1, \
1630 }) \
1631 .feat \
1632 )
1633
kvm_s390_set_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1634 static int kvm_s390_set_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1635 {
1636 struct kvm_s390_vm_cpu_uv_feat __user *ptr = (void __user *)attr->addr;
1637 unsigned long data, filter;
1638
1639 filter = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1640 if (get_user(data, &ptr->feat))
1641 return -EFAULT;
1642 if (!bitmap_subset(&data, &filter, KVM_S390_VM_CPU_UV_FEAT_NR_BITS))
1643 return -EINVAL;
1644
1645 mutex_lock(&kvm->lock);
1646 if (kvm->created_vcpus) {
1647 mutex_unlock(&kvm->lock);
1648 return -EBUSY;
1649 }
1650 kvm->arch.model.uv_feat_guest.feat = data;
1651 mutex_unlock(&kvm->lock);
1652
1653 VM_EVENT(kvm, 3, "SET: guest UV-feat: 0x%16.16lx", data);
1654
1655 return 0;
1656 }
1657
kvm_s390_set_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)1658 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1659 {
1660 int ret = -ENXIO;
1661
1662 switch (attr->attr) {
1663 case KVM_S390_VM_CPU_PROCESSOR:
1664 ret = kvm_s390_set_processor(kvm, attr);
1665 break;
1666 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1667 ret = kvm_s390_set_processor_feat(kvm, attr);
1668 break;
1669 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1670 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1671 break;
1672 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1673 ret = kvm_s390_set_uv_feat(kvm, attr);
1674 break;
1675 }
1676 return ret;
1677 }
1678
kvm_s390_get_processor(struct kvm * kvm,struct kvm_device_attr * attr)1679 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1680 {
1681 struct kvm_s390_vm_cpu_processor *proc;
1682 int ret = 0;
1683
1684 proc = kzalloc_obj(*proc, GFP_KERNEL_ACCOUNT);
1685 if (!proc) {
1686 ret = -ENOMEM;
1687 goto out;
1688 }
1689 proc->cpuid = kvm->arch.model.cpuid;
1690 proc->ibc = kvm->arch.model.ibc;
1691 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1692 S390_ARCH_FAC_LIST_SIZE_BYTE);
1693 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1694 kvm->arch.model.ibc,
1695 kvm->arch.model.cpuid);
1696 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1697 kvm->arch.model.fac_list[0],
1698 kvm->arch.model.fac_list[1],
1699 kvm->arch.model.fac_list[2]);
1700 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1701 ret = -EFAULT;
1702 kfree(proc);
1703 out:
1704 return ret;
1705 }
1706
kvm_s390_get_machine(struct kvm * kvm,struct kvm_device_attr * attr)1707 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1708 {
1709 struct kvm_s390_vm_cpu_machine *mach;
1710 int ret = 0;
1711
1712 mach = kzalloc_obj(*mach, GFP_KERNEL_ACCOUNT);
1713 if (!mach) {
1714 ret = -ENOMEM;
1715 goto out;
1716 }
1717 get_cpu_id((struct cpuid *) &mach->cpuid);
1718 mach->ibc = sclp.ibc;
1719 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1720 S390_ARCH_FAC_LIST_SIZE_BYTE);
1721 memcpy((unsigned long *)&mach->fac_list, stfle_fac_list,
1722 sizeof(stfle_fac_list));
1723 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1724 kvm->arch.model.ibc,
1725 kvm->arch.model.cpuid);
1726 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1727 mach->fac_mask[0],
1728 mach->fac_mask[1],
1729 mach->fac_mask[2]);
1730 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1731 mach->fac_list[0],
1732 mach->fac_list[1],
1733 mach->fac_list[2]);
1734 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1735 ret = -EFAULT;
1736 kfree(mach);
1737 out:
1738 return ret;
1739 }
1740
kvm_s390_get_processor_feat(struct kvm * kvm,struct kvm_device_attr * attr)1741 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1742 struct kvm_device_attr *attr)
1743 {
1744 struct kvm_s390_vm_cpu_feat data;
1745
1746 bitmap_to_arr64(data.feat, kvm->arch.cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1747 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1748 return -EFAULT;
1749 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1750 data.feat[0],
1751 data.feat[1],
1752 data.feat[2]);
1753 return 0;
1754 }
1755
kvm_s390_get_machine_feat(struct kvm * kvm,struct kvm_device_attr * attr)1756 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1757 struct kvm_device_attr *attr)
1758 {
1759 struct kvm_s390_vm_cpu_feat data;
1760
1761 bitmap_to_arr64(data.feat, kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1762 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1763 return -EFAULT;
1764 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1765 data.feat[0],
1766 data.feat[1],
1767 data.feat[2]);
1768 return 0;
1769 }
1770
kvm_s390_get_processor_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1771 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1772 struct kvm_device_attr *attr)
1773 {
1774 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1775 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1776 return -EFAULT;
1777
1778 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1779 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1780 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1781 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1782 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1783 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1784 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1785 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1786 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1787 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1788 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1789 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1790 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1791 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1792 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1793 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1794 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1795 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1796 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1797 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1798 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1799 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1800 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1801 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1802 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1803 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1804 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1805 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1806 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1807 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1808 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1809 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1810 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1811 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1812 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1813 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1814 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1815 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1816 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1817 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1818 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1819 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1820 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1821 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1822 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1823 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1824 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1825 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1826 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1827 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1828 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1829 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1830 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1831 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1832 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1833 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1834 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1835 VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1836 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1837 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1838
1839 return 0;
1840 }
1841
kvm_s390_get_machine_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1842 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1843 struct kvm_device_attr *attr)
1844 {
1845 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1846 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1847 return -EFAULT;
1848
1849 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1850 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1851 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1852 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1853 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1854 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1855 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1856 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1857 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1858 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1859 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1860 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1861 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1862 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1863 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1864 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1865 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1866 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1867 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1868 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1869 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1870 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1871 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1872 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1873 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1874 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1875 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1876 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1877 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1878 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1879 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1880 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1881 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1882 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1883 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1884 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1885 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1886 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1887 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1888 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1889 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1890 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1891 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1892 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1893 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1894 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1895 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1896 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1897 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1898 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1899 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1900 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1901 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1902 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1903 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1904 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1905 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1906 VM_EVENT(kvm, 3, "GET: host PFCR subfunc 0x%16.16lx.%16.16lx",
1907 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1908 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1909
1910 return 0;
1911 }
1912
kvm_s390_get_processor_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1913 static int kvm_s390_get_processor_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1914 {
1915 struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1916 unsigned long feat = kvm->arch.model.uv_feat_guest.feat;
1917
1918 if (put_user(feat, &dst->feat))
1919 return -EFAULT;
1920 VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1921
1922 return 0;
1923 }
1924
kvm_s390_get_machine_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1925 static int kvm_s390_get_machine_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1926 {
1927 struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1928 unsigned long feat;
1929
1930 BUILD_BUG_ON(sizeof(*dst) != sizeof(uv_info.uv_feature_indications));
1931
1932 feat = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1933 if (put_user(feat, &dst->feat))
1934 return -EFAULT;
1935 VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1936
1937 return 0;
1938 }
1939
kvm_s390_get_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)1940 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1941 {
1942 int ret = -ENXIO;
1943
1944 switch (attr->attr) {
1945 case KVM_S390_VM_CPU_PROCESSOR:
1946 ret = kvm_s390_get_processor(kvm, attr);
1947 break;
1948 case KVM_S390_VM_CPU_MACHINE:
1949 ret = kvm_s390_get_machine(kvm, attr);
1950 break;
1951 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1952 ret = kvm_s390_get_processor_feat(kvm, attr);
1953 break;
1954 case KVM_S390_VM_CPU_MACHINE_FEAT:
1955 ret = kvm_s390_get_machine_feat(kvm, attr);
1956 break;
1957 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1958 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1959 break;
1960 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1961 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1962 break;
1963 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1964 ret = kvm_s390_get_processor_uv_feat(kvm, attr);
1965 break;
1966 case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
1967 ret = kvm_s390_get_machine_uv_feat(kvm, attr);
1968 break;
1969 }
1970 return ret;
1971 }
1972
1973 /**
1974 * kvm_s390_update_topology_change_report - update CPU topology change report
1975 * @kvm: guest KVM description
1976 * @val: set or clear the MTCR bit
1977 *
1978 * Updates the Multiprocessor Topology-Change-Report bit to signal
1979 * the guest with a topology change.
1980 * This is only relevant if the topology facility is present.
1981 */
kvm_s390_update_topology_change_report(struct kvm * kvm,bool val)1982 static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
1983 {
1984 union sca_utility new, old;
1985 struct esca_block *sca;
1986
1987 sca = kvm->arch.sca;
1988 old = READ_ONCE(sca->utility);
1989 do {
1990 new = old;
1991 new.mtcr = val;
1992 } while (!try_cmpxchg(&sca->utility.val, &old.val, new.val));
1993 }
1994
kvm_s390_set_topo_change_indication(struct kvm * kvm,struct kvm_device_attr * attr)1995 static int kvm_s390_set_topo_change_indication(struct kvm *kvm,
1996 struct kvm_device_attr *attr)
1997 {
1998 if (!test_kvm_facility(kvm, 11))
1999 return -ENXIO;
2000
2001 kvm_s390_update_topology_change_report(kvm, !!attr->attr);
2002 return 0;
2003 }
2004
kvm_s390_get_topo_change_indication(struct kvm * kvm,struct kvm_device_attr * attr)2005 static int kvm_s390_get_topo_change_indication(struct kvm *kvm,
2006 struct kvm_device_attr *attr)
2007 {
2008 u8 topo;
2009
2010 if (!test_kvm_facility(kvm, 11))
2011 return -ENXIO;
2012
2013 topo = kvm->arch.sca->utility.mtcr;
2014
2015 return put_user(topo, (u8 __user *)attr->addr);
2016 }
2017
kvm_s390_vm_set_attr(struct kvm * kvm,struct kvm_device_attr * attr)2018 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2019 {
2020 int ret;
2021
2022 switch (attr->group) {
2023 case KVM_S390_VM_MEM_CTRL:
2024 ret = kvm_s390_set_mem_control(kvm, attr);
2025 break;
2026 case KVM_S390_VM_TOD:
2027 ret = kvm_s390_set_tod(kvm, attr);
2028 break;
2029 case KVM_S390_VM_CPU_MODEL:
2030 ret = kvm_s390_set_cpu_model(kvm, attr);
2031 break;
2032 case KVM_S390_VM_CRYPTO:
2033 ret = kvm_s390_vm_set_crypto(kvm, attr);
2034 break;
2035 case KVM_S390_VM_MIGRATION:
2036 ret = kvm_s390_vm_set_migration(kvm, attr);
2037 break;
2038 case KVM_S390_VM_CPU_TOPOLOGY:
2039 ret = kvm_s390_set_topo_change_indication(kvm, attr);
2040 break;
2041 default:
2042 ret = -ENXIO;
2043 break;
2044 }
2045
2046 return ret;
2047 }
2048
kvm_s390_vm_get_attr(struct kvm * kvm,struct kvm_device_attr * attr)2049 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2050 {
2051 int ret;
2052
2053 switch (attr->group) {
2054 case KVM_S390_VM_MEM_CTRL:
2055 ret = kvm_s390_get_mem_control(kvm, attr);
2056 break;
2057 case KVM_S390_VM_TOD:
2058 ret = kvm_s390_get_tod(kvm, attr);
2059 break;
2060 case KVM_S390_VM_CPU_MODEL:
2061 ret = kvm_s390_get_cpu_model(kvm, attr);
2062 break;
2063 case KVM_S390_VM_MIGRATION:
2064 ret = kvm_s390_vm_get_migration(kvm, attr);
2065 break;
2066 case KVM_S390_VM_CPU_TOPOLOGY:
2067 ret = kvm_s390_get_topo_change_indication(kvm, attr);
2068 break;
2069 default:
2070 ret = -ENXIO;
2071 break;
2072 }
2073
2074 return ret;
2075 }
2076
kvm_s390_vm_has_attr(struct kvm * kvm,struct kvm_device_attr * attr)2077 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2078 {
2079 int ret;
2080
2081 switch (attr->group) {
2082 case KVM_S390_VM_MEM_CTRL:
2083 switch (attr->attr) {
2084 case KVM_S390_VM_MEM_ENABLE_CMMA:
2085 case KVM_S390_VM_MEM_CLR_CMMA:
2086 ret = sclp.has_cmma ? 0 : -ENXIO;
2087 break;
2088 case KVM_S390_VM_MEM_LIMIT_SIZE:
2089 ret = 0;
2090 break;
2091 default:
2092 ret = -ENXIO;
2093 break;
2094 }
2095 break;
2096 case KVM_S390_VM_TOD:
2097 switch (attr->attr) {
2098 case KVM_S390_VM_TOD_LOW:
2099 case KVM_S390_VM_TOD_HIGH:
2100 ret = 0;
2101 break;
2102 default:
2103 ret = -ENXIO;
2104 break;
2105 }
2106 break;
2107 case KVM_S390_VM_CPU_MODEL:
2108 switch (attr->attr) {
2109 case KVM_S390_VM_CPU_PROCESSOR:
2110 case KVM_S390_VM_CPU_MACHINE:
2111 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
2112 case KVM_S390_VM_CPU_MACHINE_FEAT:
2113 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
2114 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
2115 case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
2116 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
2117 ret = 0;
2118 break;
2119 default:
2120 ret = -ENXIO;
2121 break;
2122 }
2123 break;
2124 case KVM_S390_VM_CRYPTO:
2125 switch (attr->attr) {
2126 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
2127 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
2128 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
2129 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
2130 ret = 0;
2131 break;
2132 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
2133 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
2134 ret = ap_instructions_available() ? 0 : -ENXIO;
2135 break;
2136 default:
2137 ret = -ENXIO;
2138 break;
2139 }
2140 break;
2141 case KVM_S390_VM_MIGRATION:
2142 ret = 0;
2143 break;
2144 case KVM_S390_VM_CPU_TOPOLOGY:
2145 ret = test_kvm_facility(kvm, 11) ? 0 : -ENXIO;
2146 break;
2147 default:
2148 ret = -ENXIO;
2149 break;
2150 }
2151
2152 return ret;
2153 }
2154
kvm_s390_get_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)2155 static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2156 {
2157 union skey *keys;
2158 int i, r = 0;
2159
2160 if (args->flags != 0)
2161 return -EINVAL;
2162
2163 /* Is this guest using storage keys? */
2164 if (!uses_skeys(kvm->arch.gmap))
2165 return KVM_S390_GET_SKEYS_NONE;
2166
2167 /* Enforce sane limit on memory allocation */
2168 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2169 return -EINVAL;
2170
2171 keys = kvmalloc_objs(*keys, args->count, GFP_KERNEL_ACCOUNT);
2172 if (!keys)
2173 return -ENOMEM;
2174
2175 scoped_guard(read_lock, &kvm->mmu_lock) {
2176 for (i = 0; i < args->count; i++) {
2177 r = dat_get_storage_key(kvm->arch.gmap->asce,
2178 args->start_gfn + i, keys + i);
2179 if (r)
2180 break;
2181 }
2182 }
2183
2184 if (!r) {
2185 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
2186 sizeof(uint8_t) * args->count);
2187 if (r)
2188 r = -EFAULT;
2189 }
2190
2191 kvfree(keys);
2192 return r <= 0 ? r : -EFAULT;
2193 }
2194
kvm_s390_set_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)2195 static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2196 {
2197 struct kvm_s390_mmu_cache *mc;
2198 union skey *keys;
2199 int i, r = 0;
2200
2201 if (args->flags != 0)
2202 return -EINVAL;
2203
2204 /* Enforce sane limit on memory allocation */
2205 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2206 return -EINVAL;
2207
2208 keys = kvmalloc_objs(*keys, args->count, GFP_KERNEL_ACCOUNT);
2209 if (!keys)
2210 return -ENOMEM;
2211
2212 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
2213 sizeof(uint8_t) * args->count);
2214 if (r) {
2215 r = -EFAULT;
2216 goto out;
2217 }
2218
2219 /* Enable storage key handling for the guest */
2220 r = gmap_enable_skeys(kvm->arch.gmap);
2221 if (r)
2222 goto out;
2223
2224 r = -EINVAL;
2225 for (i = 0; i < args->count; i++) {
2226 /* Lowest order bit is reserved */
2227 if (keys[i].zero)
2228 goto out;
2229 }
2230
2231 mc = kvm_s390_new_mmu_cache();
2232 if (!mc) {
2233 r = -ENOMEM;
2234 goto out;
2235 }
2236
2237 r = 0;
2238 do {
2239 r = kvm_s390_mmu_cache_topup(mc);
2240 if (r == -ENOMEM)
2241 break;
2242 scoped_guard(read_lock, &kvm->mmu_lock) {
2243 for (i = 0 ; i < args->count; i++) {
2244 r = dat_set_storage_key(mc, kvm->arch.gmap->asce,
2245 args->start_gfn + i, keys[i], 0);
2246 if (r)
2247 break;
2248 }
2249 }
2250 } while (r == -ENOMEM);
2251 kvm_s390_free_mmu_cache(mc);
2252 out:
2253 kvfree(keys);
2254 return r <= 0 ? r : -EFAULT;
2255 }
2256
2257 /*
2258 * This function searches for the next page with dirty CMMA attributes, and
2259 * saves the attributes in the buffer up to either the end of the buffer or
2260 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2261 * no trailing clean bytes are saved.
2262 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2263 * output buffer will indicate 0 as length.
2264 */
kvm_s390_get_cmma_bits(struct kvm * kvm,struct kvm_s390_cmma_log * args)2265 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2266 struct kvm_s390_cmma_log *args)
2267 {
2268 int peek, ret;
2269 u8 *values;
2270
2271 if (!kvm->arch.use_cmma)
2272 return -ENXIO;
2273 /* Invalid/unsupported flags were specified */
2274 if (args->flags & ~KVM_S390_CMMA_PEEK)
2275 return -EINVAL;
2276 /* Migration mode query, and we are not doing a migration */
2277 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2278 if (!peek && !kvm->arch.migration_mode)
2279 return -EINVAL;
2280 /* CMMA is disabled or was not used, or the buffer has length zero */
2281 args->count = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2282 if (!args->count || !uses_cmm(kvm->arch.gmap)) {
2283 memset(args, 0, sizeof(*args));
2284 return 0;
2285 }
2286 /* We are not peeking, and there are no dirty pages */
2287 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2288 memset(args, 0, sizeof(*args));
2289 return 0;
2290 }
2291
2292 values = vzalloc(args->count);
2293 if (!values)
2294 return -ENOMEM;
2295
2296 scoped_guard(read_lock, &kvm->mmu_lock) {
2297 if (peek)
2298 ret = dat_peek_cmma(args->start_gfn, kvm->arch.gmap->asce, &args->count,
2299 values);
2300 else
2301 ret = dat_get_cmma(kvm->arch.gmap->asce, &args->start_gfn, &args->count,
2302 values, &kvm->arch.cmma_dirty_pages);
2303 }
2304
2305 if (kvm->arch.migration_mode)
2306 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2307 else
2308 args->remaining = 0;
2309
2310 if (copy_to_user((void __user *)args->values, values, args->count))
2311 ret = -EFAULT;
2312
2313 vfree(values);
2314 return ret;
2315 }
2316
2317 /*
2318 * This function sets the CMMA attributes for the given pages. If the input
2319 * buffer has zero length, no action is taken, otherwise the attributes are
2320 * set and the mm->context.uses_cmm flag is set.
2321 */
kvm_s390_set_cmma_bits(struct kvm * kvm,const struct kvm_s390_cmma_log * args)2322 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2323 const struct kvm_s390_cmma_log *args)
2324 {
2325 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
2326 u8 *bits __free(kvfree) = NULL;
2327 int r = 0;
2328
2329 if (!kvm->arch.use_cmma)
2330 return -ENXIO;
2331 /* invalid/unsupported flags */
2332 if (args->flags != 0)
2333 return -EINVAL;
2334 /* Enforce sane limit on memory allocation */
2335 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2336 return -EINVAL;
2337 /* Nothing to do */
2338 if (args->count == 0)
2339 return 0;
2340
2341 mc = kvm_s390_new_mmu_cache();
2342 if (!mc)
2343 return -ENOMEM;
2344 bits = vmalloc(array_size(sizeof(*bits), args->count));
2345 if (!bits)
2346 return -ENOMEM;
2347
2348 r = copy_from_user(bits, (void __user *)args->values, args->count);
2349 if (r)
2350 return -EFAULT;
2351
2352 do {
2353 r = kvm_s390_mmu_cache_topup(mc);
2354 if (r)
2355 return r;
2356 scoped_guard(read_lock, &kvm->mmu_lock) {
2357 r = dat_set_cmma_bits(mc, kvm->arch.gmap->asce, args->start_gfn,
2358 args->count, args->mask, bits);
2359 }
2360 } while (r == -ENOMEM);
2361
2362 set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags);
2363
2364 return r <= 0 ? r : -EFAULT;
2365 }
2366
2367 /**
2368 * kvm_s390_cpus_from_pv - Convert all protected vCPUs in a protected VM to
2369 * non protected.
2370 * @kvm: the VM whose protected vCPUs are to be converted
2371 * @rc: return value for the RC field of the UVC (in case of error)
2372 * @rrc: return value for the RRC field of the UVC (in case of error)
2373 *
2374 * Does not stop in case of error, tries to convert as many
2375 * CPUs as possible. In case of error, the RC and RRC of the last error are
2376 * returned.
2377 *
2378 * Return: 0 in case of success, otherwise -EIO
2379 */
kvm_s390_cpus_from_pv(struct kvm * kvm,u16 * rc,u16 * rrc)2380 int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2381 {
2382 struct kvm_vcpu *vcpu;
2383 unsigned long i;
2384 u16 _rc, _rrc;
2385 int ret = 0;
2386
2387 /*
2388 * We ignore failures and try to destroy as many CPUs as possible.
2389 * At the same time we must not free the assigned resources when
2390 * this fails, as the ultravisor has still access to that memory.
2391 * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2392 * behind.
2393 * We want to return the first failure rc and rrc, though.
2394 */
2395 kvm_for_each_vcpu(i, vcpu, kvm) {
2396 mutex_lock(&vcpu->mutex);
2397 if (kvm_s390_pv_destroy_cpu(vcpu, &_rc, &_rrc) && !ret) {
2398 *rc = _rc;
2399 *rrc = _rrc;
2400 ret = -EIO;
2401 }
2402 mutex_unlock(&vcpu->mutex);
2403 }
2404 /* Ensure that we re-enable gisa if the non-PV guest used it but the PV guest did not. */
2405 if (use_gisa)
2406 kvm_s390_gisa_enable(kvm);
2407 return ret;
2408 }
2409
2410 /**
2411 * kvm_s390_cpus_to_pv - Convert all non-protected vCPUs in a protected VM
2412 * to protected.
2413 * @kvm: the VM whose protected vCPUs are to be converted
2414 * @rc: return value for the RC field of the UVC (in case of error)
2415 * @rrc: return value for the RRC field of the UVC (in case of error)
2416 *
2417 * Tries to undo the conversion in case of error.
2418 *
2419 * Return: 0 in case of success, otherwise -EIO
2420 */
kvm_s390_cpus_to_pv(struct kvm * kvm,u16 * rc,u16 * rrc)2421 static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2422 {
2423 unsigned long i;
2424 int r = 0;
2425 u16 dummy;
2426
2427 struct kvm_vcpu *vcpu;
2428
2429 /* Disable the GISA if the ultravisor does not support AIV. */
2430 if (!uv_has_feature(BIT_UV_FEAT_AIV))
2431 kvm_s390_gisa_disable(kvm);
2432
2433 kvm_for_each_vcpu(i, vcpu, kvm) {
2434 mutex_lock(&vcpu->mutex);
2435 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2436 mutex_unlock(&vcpu->mutex);
2437 if (r)
2438 break;
2439 }
2440 if (r)
2441 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2442 return r;
2443 }
2444
2445 /*
2446 * Here we provide user space with a direct interface to query UV
2447 * related data like UV maxima and available features as well as
2448 * feature specific data.
2449 *
2450 * To facilitate future extension of the data structures we'll try to
2451 * write data up to the maximum requested length.
2452 */
kvm_s390_handle_pv_info(struct kvm_s390_pv_info * info)2453 static ssize_t kvm_s390_handle_pv_info(struct kvm_s390_pv_info *info)
2454 {
2455 ssize_t len_min;
2456
2457 switch (info->header.id) {
2458 case KVM_PV_INFO_VM: {
2459 len_min = sizeof(info->header) + sizeof(info->vm);
2460
2461 if (info->header.len_max < len_min)
2462 return -EINVAL;
2463
2464 memcpy(info->vm.inst_calls_list,
2465 uv_info.inst_calls_list,
2466 sizeof(uv_info.inst_calls_list));
2467
2468 /* It's max cpuid not max cpus, so it's off by one */
2469 info->vm.max_cpus = uv_info.max_guest_cpu_id + 1;
2470 info->vm.max_guests = uv_info.max_num_sec_conf;
2471 info->vm.max_guest_addr = uv_info.max_sec_stor_addr;
2472 info->vm.feature_indication = uv_info.uv_feature_indications;
2473
2474 return len_min;
2475 }
2476 case KVM_PV_INFO_DUMP: {
2477 len_min = sizeof(info->header) + sizeof(info->dump);
2478
2479 if (info->header.len_max < len_min)
2480 return -EINVAL;
2481
2482 info->dump.dump_cpu_buffer_len = uv_info.guest_cpu_stor_len;
2483 info->dump.dump_config_mem_buffer_per_1m = uv_info.conf_dump_storage_state_len;
2484 info->dump.dump_config_finalize_len = uv_info.conf_dump_finalize_len;
2485 return len_min;
2486 }
2487 default:
2488 return -EINVAL;
2489 }
2490 }
2491
kvm_s390_pv_dmp(struct kvm * kvm,struct kvm_pv_cmd * cmd,struct kvm_s390_pv_dmp dmp)2492 static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd,
2493 struct kvm_s390_pv_dmp dmp)
2494 {
2495 int r = -EINVAL;
2496 void __user *result_buff = (void __user *)dmp.buff_addr;
2497
2498 switch (dmp.subcmd) {
2499 case KVM_PV_DUMP_INIT: {
2500 if (kvm->arch.pv.dumping)
2501 break;
2502
2503 /*
2504 * Block SIE entry as concurrent dump UVCs could lead
2505 * to validities.
2506 */
2507 kvm_s390_vcpu_block_all(kvm);
2508
2509 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2510 UVC_CMD_DUMP_INIT, &cmd->rc, &cmd->rrc);
2511 KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP INIT: rc %x rrc %x",
2512 cmd->rc, cmd->rrc);
2513 if (!r) {
2514 kvm->arch.pv.dumping = true;
2515 } else {
2516 kvm_s390_vcpu_unblock_all(kvm);
2517 r = -EINVAL;
2518 }
2519 break;
2520 }
2521 case KVM_PV_DUMP_CONFIG_STOR_STATE: {
2522 if (!kvm->arch.pv.dumping)
2523 break;
2524
2525 /*
2526 * gaddr is an output parameter since we might stop
2527 * early. As dmp will be copied back in our caller, we
2528 * don't need to do it ourselves.
2529 */
2530 r = kvm_s390_pv_dump_stor_state(kvm, result_buff, &dmp.gaddr, dmp.buff_len,
2531 &cmd->rc, &cmd->rrc);
2532 break;
2533 }
2534 case KVM_PV_DUMP_COMPLETE: {
2535 if (!kvm->arch.pv.dumping)
2536 break;
2537
2538 r = -EINVAL;
2539 if (dmp.buff_len < uv_info.conf_dump_finalize_len)
2540 break;
2541
2542 r = kvm_s390_pv_dump_complete(kvm, result_buff,
2543 &cmd->rc, &cmd->rrc);
2544 break;
2545 }
2546 default:
2547 r = -ENOTTY;
2548 break;
2549 }
2550
2551 return r;
2552 }
2553
kvm_s390_handle_pv(struct kvm * kvm,struct kvm_pv_cmd * cmd)2554 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2555 {
2556 const bool need_lock = (cmd->cmd != KVM_PV_ASYNC_CLEANUP_PERFORM);
2557 void __user *argp = (void __user *)cmd->data;
2558 int r = 0;
2559 u16 dummy;
2560
2561 if (need_lock)
2562 mutex_lock(&kvm->lock);
2563
2564 switch (cmd->cmd) {
2565 case KVM_PV_ENABLE: {
2566 r = -EINVAL;
2567 if (kvm_s390_pv_is_protected(kvm))
2568 break;
2569
2570 kvm_s390_unmap_all_adapters(kvm);
2571 mmap_write_lock(kvm->mm);
2572 /*
2573 * Disable creation of new THPs. Existing THPs can stay, they
2574 * will be split when any part of them gets imported.
2575 */
2576 mm_flags_clear(MMF_DISABLE_THP_EXCEPT_ADVISED, kvm->mm);
2577 mm_flags_set(MMF_DISABLE_THP_COMPLETELY, kvm->mm);
2578 set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
2579 r = gmap_helper_disable_cow_sharing();
2580 mmap_write_unlock(kvm->mm);
2581 if (r)
2582 break;
2583
2584 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2585 if (r)
2586 break;
2587
2588 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2589 if (r)
2590 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
2591
2592 /* we need to block service interrupts from now on */
2593 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2594 break;
2595 }
2596 case KVM_PV_ASYNC_CLEANUP_PREPARE:
2597 r = -EINVAL;
2598 if (!kvm_s390_pv_is_protected(kvm) || !async_destroy)
2599 break;
2600
2601 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2602 /*
2603 * If a CPU could not be destroyed, destroy VM will also fail.
2604 * There is no point in trying to destroy it. Instead return
2605 * the rc and rrc from the first CPU that failed destroying.
2606 */
2607 if (r)
2608 break;
2609 r = kvm_s390_pv_set_aside(kvm, &cmd->rc, &cmd->rrc);
2610
2611 /* no need to block service interrupts any more */
2612 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2613 break;
2614 case KVM_PV_ASYNC_CLEANUP_PERFORM:
2615 r = -EINVAL;
2616 if (!async_destroy)
2617 break;
2618 /* kvm->lock must not be held; this is asserted inside the function. */
2619 r = kvm_s390_pv_deinit_aside_vm(kvm, &cmd->rc, &cmd->rrc);
2620 break;
2621 case KVM_PV_DISABLE: {
2622 r = -EINVAL;
2623 if (!kvm_s390_pv_is_protected(kvm))
2624 break;
2625
2626 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2627 /*
2628 * If a CPU could not be destroyed, destroy VM will also fail.
2629 * There is no point in trying to destroy it. Instead return
2630 * the rc and rrc from the first CPU that failed destroying.
2631 */
2632 if (r)
2633 break;
2634 r = kvm_s390_pv_deinit_cleanup_all(kvm, &cmd->rc, &cmd->rrc);
2635
2636 /* no need to block service interrupts any more */
2637 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2638 break;
2639 }
2640 case KVM_PV_SET_SEC_PARMS: {
2641 struct kvm_s390_pv_sec_parm parms = {};
2642 void *hdr;
2643
2644 r = -EINVAL;
2645 if (!kvm_s390_pv_is_protected(kvm))
2646 break;
2647
2648 r = -EFAULT;
2649 if (copy_from_user(&parms, argp, sizeof(parms)))
2650 break;
2651
2652 /* Currently restricted to 1MiB */
2653 r = -EINVAL;
2654 if (parms.length > SZ_1M)
2655 break;
2656
2657 r = -ENOMEM;
2658 hdr = vmalloc(parms.length);
2659 if (!hdr)
2660 break;
2661
2662 r = -EFAULT;
2663 if (!copy_from_user(hdr, (void __user *)parms.origin,
2664 parms.length))
2665 r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2666 &cmd->rc, &cmd->rrc);
2667
2668 vfree(hdr);
2669 break;
2670 }
2671 case KVM_PV_UNPACK: {
2672 struct kvm_s390_pv_unp unp = {};
2673
2674 r = -EINVAL;
2675 if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
2676 break;
2677
2678 r = -EFAULT;
2679 if (copy_from_user(&unp, argp, sizeof(unp)))
2680 break;
2681
2682 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2683 &cmd->rc, &cmd->rrc);
2684 break;
2685 }
2686 case KVM_PV_VERIFY: {
2687 r = -EINVAL;
2688 if (!kvm_s390_pv_is_protected(kvm))
2689 break;
2690
2691 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2692 UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2693 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2694 cmd->rrc);
2695 break;
2696 }
2697 case KVM_PV_PREP_RESET: {
2698 r = -EINVAL;
2699 if (!kvm_s390_pv_is_protected(kvm))
2700 break;
2701
2702 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2703 UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2704 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2705 cmd->rc, cmd->rrc);
2706 break;
2707 }
2708 case KVM_PV_UNSHARE_ALL: {
2709 r = -EINVAL;
2710 if (!kvm_s390_pv_is_protected(kvm))
2711 break;
2712
2713 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2714 UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2715 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2716 cmd->rc, cmd->rrc);
2717 break;
2718 }
2719 case KVM_PV_INFO: {
2720 struct kvm_s390_pv_info info = {};
2721 ssize_t data_len;
2722
2723 /*
2724 * No need to check the VM protection here.
2725 *
2726 * Maybe user space wants to query some of the data
2727 * when the VM is still unprotected. If we see the
2728 * need to fence a new data command we can still
2729 * return an error in the info handler.
2730 */
2731
2732 r = -EFAULT;
2733 if (copy_from_user(&info, argp, sizeof(info.header)))
2734 break;
2735
2736 r = -EINVAL;
2737 if (info.header.len_max < sizeof(info.header))
2738 break;
2739
2740 data_len = kvm_s390_handle_pv_info(&info);
2741 if (data_len < 0) {
2742 r = data_len;
2743 break;
2744 }
2745 /*
2746 * If a data command struct is extended (multiple
2747 * times) this can be used to determine how much of it
2748 * is valid.
2749 */
2750 info.header.len_written = data_len;
2751
2752 r = -EFAULT;
2753 if (copy_to_user(argp, &info, data_len))
2754 break;
2755
2756 r = 0;
2757 break;
2758 }
2759 case KVM_PV_DUMP: {
2760 struct kvm_s390_pv_dmp dmp;
2761
2762 r = -EINVAL;
2763 if (!kvm_s390_pv_is_protected(kvm))
2764 break;
2765
2766 r = -EFAULT;
2767 if (copy_from_user(&dmp, argp, sizeof(dmp)))
2768 break;
2769
2770 r = kvm_s390_pv_dmp(kvm, cmd, dmp);
2771 if (r)
2772 break;
2773
2774 if (copy_to_user(argp, &dmp, sizeof(dmp))) {
2775 r = -EFAULT;
2776 break;
2777 }
2778
2779 break;
2780 }
2781 default:
2782 r = -ENOTTY;
2783 }
2784 if (need_lock)
2785 mutex_unlock(&kvm->lock);
2786
2787 return r;
2788 }
2789
mem_op_validate_common(struct kvm_s390_mem_op * mop,u64 supported_flags)2790 static int mem_op_validate_common(struct kvm_s390_mem_op *mop, u64 supported_flags)
2791 {
2792 if (mop->flags & ~supported_flags || !mop->size)
2793 return -EINVAL;
2794 if (mop->size > MEM_OP_MAX_SIZE)
2795 return -E2BIG;
2796 if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
2797 if (mop->key > 0xf)
2798 return -EINVAL;
2799 } else {
2800 mop->key = 0;
2801 }
2802 return 0;
2803 }
2804
kvm_s390_vm_mem_op_abs(struct kvm * kvm,struct kvm_s390_mem_op * mop)2805 static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2806 {
2807 void __user *uaddr = (void __user *)mop->buf;
2808 void *tmpbuf __free(kvfree) = NULL;
2809 enum gacc_mode acc_mode;
2810 int r;
2811
2812 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION |
2813 KVM_S390_MEMOP_F_CHECK_ONLY);
2814 if (r)
2815 return r;
2816
2817 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2818 tmpbuf = vmalloc(mop->size);
2819 if (!tmpbuf)
2820 return -ENOMEM;
2821 }
2822
2823 acc_mode = mop->op == KVM_S390_MEMOP_ABSOLUTE_READ ? GACC_FETCH : GACC_STORE;
2824
2825 scoped_guard(srcu, &kvm->srcu) {
2826 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)
2827 return check_gpa_range(kvm, mop->gaddr, mop->size, acc_mode, mop->key);
2828
2829 if (acc_mode == GACC_STORE && copy_from_user(tmpbuf, uaddr, mop->size))
2830 return -EFAULT;
2831 r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
2832 mop->size, acc_mode, mop->key);
2833 if (r)
2834 return r;
2835 if (acc_mode != GACC_STORE && copy_to_user(uaddr, tmpbuf, mop->size))
2836 return -EFAULT;
2837 }
2838 return 0;
2839 }
2840
kvm_s390_vm_mem_op_cmpxchg(struct kvm * kvm,struct kvm_s390_mem_op * mop)2841 static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2842 {
2843 void __user *uaddr = (void __user *)mop->buf;
2844 void __user *old_addr = (void __user *)mop->old_addr;
2845 union kvm_s390_quad old = { .sixteen = 0 };
2846 union kvm_s390_quad new = { .sixteen = 0 };
2847 bool success = false;
2848 int r;
2849
2850 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION);
2851 if (r)
2852 return r;
2853 /*
2854 * This validates off_in_quad. Checking that size is a power
2855 * of two is not necessary, as cmpxchg_guest_abs_with_key
2856 * takes care of that
2857 */
2858 if (mop->size > sizeof(new))
2859 return -EINVAL;
2860 if (copy_from_user(&new, uaddr, mop->size))
2861 return -EFAULT;
2862 if (copy_from_user(&old, old_addr, mop->size))
2863 return -EFAULT;
2864
2865 scoped_guard(srcu, &kvm->srcu) {
2866 r = cmpxchg_guest_abs_with_key(kvm, mop->gaddr, mop->size, &old, new,
2867 mop->key, &success);
2868
2869 if (!success && copy_to_user(old_addr, &old, mop->size))
2870 return -EFAULT;
2871 }
2872 return r;
2873 }
2874
kvm_s390_vm_mem_op(struct kvm * kvm,struct kvm_s390_mem_op * mop)2875 static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2876 {
2877 /*
2878 * This is technically a heuristic only, if the kvm->lock is not
2879 * taken, it is not guaranteed that the vm is/remains non-protected.
2880 * This is ok from a kernel perspective, wrongdoing is detected
2881 * on the access, -EFAULT is returned and the vm may crash the
2882 * next time it accesses the memory in question.
2883 * There is no sane usecase to do switching and a memop on two
2884 * different CPUs at the same time.
2885 */
2886 if (kvm_s390_pv_get_handle(kvm))
2887 return -EINVAL;
2888
2889 switch (mop->op) {
2890 case KVM_S390_MEMOP_ABSOLUTE_READ:
2891 case KVM_S390_MEMOP_ABSOLUTE_WRITE:
2892 return kvm_s390_vm_mem_op_abs(kvm, mop);
2893 case KVM_S390_MEMOP_ABSOLUTE_CMPXCHG:
2894 return kvm_s390_vm_mem_op_cmpxchg(kvm, mop);
2895 default:
2896 return -EINVAL;
2897 }
2898 }
2899
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)2900 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
2901 {
2902 struct kvm *kvm = filp->private_data;
2903 void __user *argp = (void __user *)arg;
2904 struct kvm_device_attr attr;
2905 int r;
2906 struct kvm_s390_interrupt_info *inti;
2907
2908 switch (ioctl) {
2909 case KVM_S390_INTERRUPT: {
2910 struct kvm_s390_interrupt s390int;
2911
2912 r = -EINVAL;
2913 if (kvm_is_ucontrol(kvm))
2914 break;
2915 r = -EFAULT;
2916 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2917 break;
2918 inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT);
2919 if (!inti)
2920 return -ENOMEM;
2921 r = kvm_s390_inject_vm(kvm, &s390int, inti);
2922 if (r)
2923 kfree(inti);
2924 break;
2925 }
2926 case KVM_CREATE_IRQCHIP: {
2927 r = -EINVAL;
2928 if (kvm->arch.use_irqchip)
2929 r = 0;
2930 break;
2931 }
2932 case KVM_SET_DEVICE_ATTR: {
2933 r = -EFAULT;
2934 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2935 break;
2936 r = kvm_s390_vm_set_attr(kvm, &attr);
2937 break;
2938 }
2939 case KVM_GET_DEVICE_ATTR: {
2940 r = -EFAULT;
2941 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2942 break;
2943 r = kvm_s390_vm_get_attr(kvm, &attr);
2944 break;
2945 }
2946 case KVM_HAS_DEVICE_ATTR: {
2947 r = -EFAULT;
2948 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2949 break;
2950 r = kvm_s390_vm_has_attr(kvm, &attr);
2951 break;
2952 }
2953 case KVM_S390_GET_SKEYS: {
2954 struct kvm_s390_skeys args;
2955
2956 r = -EFAULT;
2957 if (copy_from_user(&args, argp,
2958 sizeof(struct kvm_s390_skeys)))
2959 break;
2960 r = kvm_s390_get_skeys(kvm, &args);
2961 break;
2962 }
2963 case KVM_S390_SET_SKEYS: {
2964 struct kvm_s390_skeys args;
2965
2966 r = -EFAULT;
2967 if (copy_from_user(&args, argp,
2968 sizeof(struct kvm_s390_skeys)))
2969 break;
2970 r = kvm_s390_set_skeys(kvm, &args);
2971 break;
2972 }
2973 case KVM_S390_GET_CMMA_BITS: {
2974 struct kvm_s390_cmma_log args;
2975
2976 r = -EFAULT;
2977 if (copy_from_user(&args, argp, sizeof(args)))
2978 break;
2979 scoped_guard(mutex, &kvm->slots_arch_lock)
2980 r = kvm_s390_get_cmma_bits(kvm, &args);
2981 if (!r) {
2982 r = copy_to_user(argp, &args, sizeof(args));
2983 if (r)
2984 r = -EFAULT;
2985 }
2986 break;
2987 }
2988 case KVM_S390_SET_CMMA_BITS: {
2989 struct kvm_s390_cmma_log args;
2990
2991 r = -EFAULT;
2992 if (copy_from_user(&args, argp, sizeof(args)))
2993 break;
2994 mutex_lock(&kvm->slots_arch_lock);
2995 r = kvm_s390_set_cmma_bits(kvm, &args);
2996 mutex_unlock(&kvm->slots_arch_lock);
2997 break;
2998 }
2999 case KVM_S390_PV_COMMAND: {
3000 struct kvm_pv_cmd args;
3001
3002 /* protvirt means user cpu state */
3003 kvm_s390_set_user_cpu_state_ctrl(kvm);
3004 r = 0;
3005 if (!is_prot_virt_host()) {
3006 r = -EINVAL;
3007 break;
3008 }
3009 if (copy_from_user(&args, argp, sizeof(args))) {
3010 r = -EFAULT;
3011 break;
3012 }
3013 if (args.flags) {
3014 r = -EINVAL;
3015 break;
3016 }
3017 /* must be called without kvm->lock */
3018 r = kvm_s390_handle_pv(kvm, &args);
3019 if (copy_to_user(argp, &args, sizeof(args))) {
3020 r = -EFAULT;
3021 break;
3022 }
3023 break;
3024 }
3025 case KVM_S390_MEM_OP: {
3026 struct kvm_s390_mem_op mem_op;
3027
3028 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3029 r = kvm_s390_vm_mem_op(kvm, &mem_op);
3030 else
3031 r = -EFAULT;
3032 break;
3033 }
3034 case KVM_S390_KEYOP: {
3035 struct kvm_s390_mmu_cache *mc;
3036 struct kvm_s390_keyop kop;
3037 union skey skey;
3038
3039 if (copy_from_user(&kop, argp, sizeof(kop))) {
3040 r = -EFAULT;
3041 break;
3042 }
3043 skey.skey = kop.key;
3044
3045 mc = kvm_s390_new_mmu_cache();
3046 if (!mc)
3047 return -ENOMEM;
3048
3049 r = kvm_s390_keyop(mc, kvm, kop.operation, kop.guest_addr, skey);
3050 kvm_s390_free_mmu_cache(mc);
3051 if (r < 0)
3052 break;
3053
3054 kop.key = r;
3055 r = 0;
3056 if (copy_to_user(argp, &kop, sizeof(kop)))
3057 r = -EFAULT;
3058 break;
3059 }
3060 case KVM_S390_ZPCI_OP: {
3061 struct kvm_s390_zpci_op args;
3062
3063 r = -EINVAL;
3064 if (!IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3065 break;
3066 if (copy_from_user(&args, argp, sizeof(args))) {
3067 r = -EFAULT;
3068 break;
3069 }
3070 r = kvm_s390_pci_zpci_op(kvm, &args);
3071 break;
3072 }
3073 default:
3074 r = -ENOTTY;
3075 }
3076
3077 return r;
3078 }
3079
kvm_s390_apxa_installed(void)3080 static int kvm_s390_apxa_installed(void)
3081 {
3082 struct ap_config_info info;
3083
3084 if (ap_instructions_available()) {
3085 if (ap_qci(&info) == 0)
3086 return info.apxa;
3087 }
3088
3089 return 0;
3090 }
3091
3092 /*
3093 * The format of the crypto control block (CRYCB) is specified in the 3 low
3094 * order bits of the CRYCB designation (CRYCBD) field as follows:
3095 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
3096 * AP extended addressing (APXA) facility are installed.
3097 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
3098 * Format 2: Both the APXA and MSAX3 facilities are installed
3099 */
kvm_s390_set_crycb_format(struct kvm * kvm)3100 static void kvm_s390_set_crycb_format(struct kvm *kvm)
3101 {
3102 kvm->arch.crypto.crycbd = virt_to_phys(kvm->arch.crypto.crycb);
3103
3104 /* Clear the CRYCB format bits - i.e., set format 0 by default */
3105 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
3106
3107 /* Check whether MSAX3 is installed */
3108 if (!test_kvm_facility(kvm, 76))
3109 return;
3110
3111 if (kvm_s390_apxa_installed())
3112 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
3113 else
3114 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
3115 }
3116
3117 /*
3118 * kvm_arch_crypto_set_masks
3119 *
3120 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
3121 * to be set.
3122 * @apm: the mask identifying the accessible AP adapters
3123 * @aqm: the mask identifying the accessible AP domains
3124 * @adm: the mask identifying the accessible AP control domains
3125 *
3126 * Set the masks that identify the adapters, domains and control domains to
3127 * which the KVM guest is granted access.
3128 *
3129 * Note: The kvm->lock mutex must be locked by the caller before invoking this
3130 * function.
3131 */
kvm_arch_crypto_set_masks(struct kvm * kvm,unsigned long * apm,unsigned long * aqm,unsigned long * adm)3132 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
3133 unsigned long *aqm, unsigned long *adm)
3134 {
3135 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
3136
3137 kvm_s390_vcpu_block_all(kvm);
3138
3139 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
3140 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
3141 memcpy(crycb->apcb1.apm, apm, 32);
3142 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
3143 apm[0], apm[1], apm[2], apm[3]);
3144 memcpy(crycb->apcb1.aqm, aqm, 32);
3145 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
3146 aqm[0], aqm[1], aqm[2], aqm[3]);
3147 memcpy(crycb->apcb1.adm, adm, 32);
3148 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
3149 adm[0], adm[1], adm[2], adm[3]);
3150 break;
3151 case CRYCB_FORMAT1:
3152 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
3153 memcpy(crycb->apcb0.apm, apm, 8);
3154 memcpy(crycb->apcb0.aqm, aqm, 2);
3155 memcpy(crycb->apcb0.adm, adm, 2);
3156 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
3157 apm[0], *((unsigned short *)aqm),
3158 *((unsigned short *)adm));
3159 break;
3160 default: /* Can not happen */
3161 break;
3162 }
3163
3164 /* recreate the shadow crycb for each vcpu */
3165 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3166 kvm_s390_vcpu_unblock_all(kvm);
3167 }
3168 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
3169
3170 /*
3171 * kvm_arch_crypto_clear_masks
3172 *
3173 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
3174 * to be cleared.
3175 *
3176 * Clear the masks that identify the adapters, domains and control domains to
3177 * which the KVM guest is granted access.
3178 *
3179 * Note: The kvm->lock mutex must be locked by the caller before invoking this
3180 * function.
3181 */
kvm_arch_crypto_clear_masks(struct kvm * kvm)3182 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
3183 {
3184 kvm_s390_vcpu_block_all(kvm);
3185
3186 memset(&kvm->arch.crypto.crycb->apcb0, 0,
3187 sizeof(kvm->arch.crypto.crycb->apcb0));
3188 memset(&kvm->arch.crypto.crycb->apcb1, 0,
3189 sizeof(kvm->arch.crypto.crycb->apcb1));
3190
3191 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
3192 /* recreate the shadow crycb for each vcpu */
3193 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3194 kvm_s390_vcpu_unblock_all(kvm);
3195 }
3196 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
3197
kvm_s390_get_initial_cpuid(void)3198 static u64 kvm_s390_get_initial_cpuid(void)
3199 {
3200 struct cpuid cpuid;
3201
3202 get_cpu_id(&cpuid);
3203 cpuid.version = 0xff;
3204 return *((u64 *) &cpuid);
3205 }
3206
kvm_s390_crypto_init(struct kvm * kvm)3207 static void kvm_s390_crypto_init(struct kvm *kvm)
3208 {
3209 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
3210 kvm_s390_set_crycb_format(kvm);
3211 init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
3212
3213 if (!test_kvm_facility(kvm, 76))
3214 return;
3215
3216 /* Enable AES/DEA protected key functions by default */
3217 kvm->arch.crypto.aes_kw = 1;
3218 kvm->arch.crypto.dea_kw = 1;
3219 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
3220 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
3221 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
3222 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
3223 }
3224
sca_dispose(struct kvm * kvm)3225 static void sca_dispose(struct kvm *kvm)
3226 {
3227 if (kvm->arch.sca)
3228 free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca));
3229 kvm->arch.sca = NULL;
3230 }
3231
kvm_arch_free_vm(struct kvm * kvm)3232 void kvm_arch_free_vm(struct kvm *kvm)
3233 {
3234 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3235 kvm_s390_pci_clear_list(kvm);
3236
3237 __kvm_arch_free_vm(kvm);
3238 }
3239
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)3240 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
3241 {
3242 gfp_t alloc_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
3243 char debug_name[16];
3244 int i, rc;
3245
3246 mutex_init(&kvm->arch.pv.import_lock);
3247
3248 rc = -EINVAL;
3249 #ifdef CONFIG_KVM_S390_UCONTROL
3250 if (type & ~KVM_VM_S390_UCONTROL)
3251 goto out_err;
3252 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
3253 goto out_err;
3254 #else
3255 if (type)
3256 goto out_err;
3257 #endif
3258 rc = -ENOMEM;
3259
3260 if (!sclp.has_64bscao)
3261 alloc_flags |= GFP_DMA;
3262 mutex_lock(&kvm_lock);
3263
3264 kvm->arch.sca = alloc_pages_exact(sizeof(*kvm->arch.sca), alloc_flags);
3265 mutex_unlock(&kvm_lock);
3266 if (!kvm->arch.sca)
3267 goto out_err;
3268
3269 snprintf(debug_name, sizeof(debug_name), "kvm-%u", current->pid);
3270
3271 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
3272 if (!kvm->arch.dbf)
3273 goto out_err;
3274
3275 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
3276 kvm->arch.sie_page2 =
3277 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3278 if (!kvm->arch.sie_page2)
3279 goto out_err;
3280
3281 kvm->arch.sie_page2->kvm = kvm;
3282 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
3283
3284 for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_base); i++) {
3285 kvm->arch.model.fac_mask[i] = stfle_fac_list[i] &
3286 kvm_s390_fac_base[i];
3287 kvm->arch.model.fac_list[i] = stfle_fac_list[i] &
3288 kvm_s390_fac_base[i];
3289 }
3290 for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_ext); i++) {
3291 kvm->arch.model.fac_mask[i] |= stfle_fac_list[i] &
3292 kvm_s390_fac_ext[i];
3293 }
3294 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
3295
3296 /* we are always in czam mode - even on pre z14 machines */
3297 set_kvm_facility(kvm->arch.model.fac_mask, 138);
3298 set_kvm_facility(kvm->arch.model.fac_list, 138);
3299 /* we emulate STHYI in kvm */
3300 set_kvm_facility(kvm->arch.model.fac_mask, 74);
3301 set_kvm_facility(kvm->arch.model.fac_list, 74);
3302 if (machine_has_tlb_guest()) {
3303 set_kvm_facility(kvm->arch.model.fac_mask, 147);
3304 set_kvm_facility(kvm->arch.model.fac_list, 147);
3305 }
3306
3307 if (css_general_characteristics.aiv && test_facility(65))
3308 set_kvm_facility(kvm->arch.model.fac_mask, 65);
3309
3310 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
3311 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
3312
3313 kvm->arch.model.uv_feat_guest.feat = 0;
3314
3315 kvm_s390_crypto_init(kvm);
3316
3317 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3318 mutex_lock(&kvm->lock);
3319 kvm_s390_pci_init_list(kvm);
3320 kvm_s390_vcpu_pci_enable_interp(kvm);
3321 mutex_unlock(&kvm->lock);
3322 }
3323
3324 spin_lock_init(&kvm->arch.float_int.ais_lock);
3325 spin_lock_init(&kvm->arch.float_int.lock);
3326 for (i = 0; i < FIRQ_LIST_COUNT; i++)
3327 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
3328 init_waitqueue_head(&kvm->arch.ipte_wq);
3329 mutex_init(&kvm->arch.ipte_mutex);
3330
3331 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
3332 VM_EVENT(kvm, 3, "vm created with type %lu", type);
3333
3334 kvm->arch.mem_limit = type & KVM_VM_S390_UCONTROL ? KVM_S390_NO_MEM_LIMIT : sclp.hamax + 1;
3335 kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.mem_limit));
3336 if (!kvm->arch.gmap)
3337 goto out_err;
3338 clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags);
3339
3340 if (type & KVM_VM_S390_UCONTROL) {
3341 struct kvm_userspace_memory_region2 fake_memslot = {
3342 .slot = KVM_S390_UCONTROL_MEMSLOT,
3343 .guest_phys_addr = 0,
3344 .userspace_addr = 0,
3345 .memory_size = ALIGN_DOWN(TASK_SIZE, _SEGMENT_SIZE),
3346 .flags = 0,
3347 };
3348
3349 /* one flat fake memslot covering the whole address-space */
3350 mutex_lock(&kvm->slots_lock);
3351 KVM_BUG_ON(kvm_set_internal_memslot(kvm, &fake_memslot), kvm);
3352 mutex_unlock(&kvm->slots_lock);
3353 set_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags);
3354 } else {
3355 struct crst_table *table = dereference_asce(kvm->arch.gmap->asce);
3356
3357 crst_table_init((void *)table, _CRSTE_HOLE(table->crstes[0].h.tt).val);
3358 }
3359
3360 kvm->arch.use_pfmfi = sclp.has_pfmfi;
3361 kvm->arch.use_skf = sclp.has_skey;
3362 spin_lock_init(&kvm->arch.start_stop_lock);
3363 kvm_s390_vsie_init(kvm);
3364 if (use_gisa)
3365 kvm_s390_gisa_init(kvm);
3366 INIT_LIST_HEAD(&kvm->arch.pv.need_cleanup);
3367 kvm->arch.pv.set_aside = NULL;
3368 KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
3369
3370 return 0;
3371 out_err:
3372 free_page((unsigned long)kvm->arch.sie_page2);
3373 debug_unregister(kvm->arch.dbf);
3374 sca_dispose(kvm);
3375 KVM_EVENT(3, "creation of vm failed: %d", rc);
3376 return rc;
3377 }
3378
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)3379 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
3380 {
3381 u16 rc, rrc;
3382
3383 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
3384 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
3385 kvm_s390_clear_local_irqs(vcpu);
3386 kvm_clear_async_pf_completion_queue(vcpu);
3387 kvm_s390_clear_bp_data(vcpu);
3388 if (!kvm_is_ucontrol(vcpu->kvm))
3389 sca_del_vcpu(vcpu);
3390 kvm_s390_update_topology_change_report(vcpu->kvm, 1);
3391
3392 if (kvm_is_ucontrol(vcpu->kvm)) {
3393 scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
3394 gmap_remove_child(vcpu->arch.gmap);
3395 vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
3396 }
3397
3398 if (vcpu->kvm->arch.use_cmma)
3399 kvm_s390_vcpu_unsetup_cmma(vcpu);
3400 /* We can not hold the vcpu mutex here, we are already dying */
3401 if (kvm_s390_pv_cpu_get_handle(vcpu))
3402 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
3403 free_page((unsigned long)(vcpu->arch.sie_block));
3404 kvm_s390_free_mmu_cache(vcpu->arch.mc);
3405 }
3406
kvm_arch_destroy_vm(struct kvm * kvm)3407 void kvm_arch_destroy_vm(struct kvm *kvm)
3408 {
3409 u16 rc, rrc;
3410
3411 kvm_destroy_vcpus(kvm);
3412 sca_dispose(kvm);
3413 kvm_s390_gisa_destroy(kvm);
3414 /*
3415 * We are already at the end of life and kvm->lock is not taken.
3416 * This is ok as the file descriptor is closed by now and nobody
3417 * can mess with the pv state.
3418 */
3419 kvm_s390_pv_deinit_cleanup_all(kvm, &rc, &rrc);
3420 /*
3421 * Remove the mmu notifier only when the whole KVM VM is torn down,
3422 * and only if one was registered to begin with. If the VM is
3423 * currently not protected, but has been previously been protected,
3424 * then it's possible that the notifier is still registered.
3425 */
3426 if (kvm->arch.pv.mmu_notifier.ops)
3427 mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm);
3428
3429 debug_unregister(kvm->arch.dbf);
3430 free_page((unsigned long)kvm->arch.sie_page2);
3431 kvm_s390_destroy_adapters(kvm);
3432 kvm_s390_clear_float_irqs(kvm);
3433 kvm_s390_vsie_destroy(kvm);
3434 kvm->arch.gmap = gmap_put(kvm->arch.gmap);
3435 KVM_EVENT(3, "vm 0x%p destroyed", kvm);
3436 }
3437
3438 /* Section: vcpu related */
sca_del_vcpu(struct kvm_vcpu * vcpu)3439 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
3440 {
3441 struct esca_block *sca = vcpu->kvm->arch.sca;
3442
3443 if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized)
3444 return;
3445
3446 clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3447 sca->cpu[vcpu->vcpu_id].sda = 0;
3448 }
3449
sca_add_vcpu(struct kvm_vcpu * vcpu)3450 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
3451 {
3452 struct esca_block *sca = vcpu->kvm->arch.sca;
3453 phys_addr_t sca_phys = virt_to_phys(sca);
3454
3455 /* we still need the sca header for the ipte control */
3456 vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3457 vcpu->arch.sie_block->scaol = sca_phys & ESCA_SCAOL_MASK;
3458 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
3459
3460 if (!kvm_s390_use_sca_entries())
3461 return;
3462
3463 WRITE_ONCE(sca->cpu[vcpu->vcpu_id].sda, virt_to_phys(vcpu->arch.sie_block));
3464 set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3465 }
3466
sca_can_add_vcpu(struct kvm * kvm,unsigned int id)3467 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
3468 {
3469 if (!kvm_s390_use_sca_entries())
3470 return id < KVM_MAX_VCPUS;
3471
3472 return id < KVM_S390_ESCA_CPU_SLOTS;
3473 }
3474
3475 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__start_cpu_timer_accounting(struct kvm_vcpu * vcpu)3476 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3477 {
3478 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
3479 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3480 vcpu->arch.cputm_start = get_tod_clock_fast();
3481 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3482 }
3483
3484 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__stop_cpu_timer_accounting(struct kvm_vcpu * vcpu)3485 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3486 {
3487 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
3488 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3489 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3490 vcpu->arch.cputm_start = 0;
3491 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3492 }
3493
3494 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__enable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3495 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3496 {
3497 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
3498 vcpu->arch.cputm_enabled = true;
3499 __start_cpu_timer_accounting(vcpu);
3500 }
3501
3502 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__disable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3503 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3504 {
3505 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
3506 __stop_cpu_timer_accounting(vcpu);
3507 vcpu->arch.cputm_enabled = false;
3508 }
3509
enable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3510 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3511 {
3512 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3513 __enable_cpu_timer_accounting(vcpu);
3514 preempt_enable();
3515 }
3516
disable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3517 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3518 {
3519 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3520 __disable_cpu_timer_accounting(vcpu);
3521 preempt_enable();
3522 }
3523
3524 /* set the cpu timer - may only be called from the VCPU thread itself */
kvm_s390_set_cpu_timer(struct kvm_vcpu * vcpu,__u64 cputm)3525 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3526 {
3527 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3528 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3529 if (vcpu->arch.cputm_enabled)
3530 vcpu->arch.cputm_start = get_tod_clock_fast();
3531 vcpu->arch.sie_block->cputm = cputm;
3532 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3533 preempt_enable();
3534 }
3535
3536 /* update and get the cpu timer - can also be called from other VCPU threads */
kvm_s390_get_cpu_timer(struct kvm_vcpu * vcpu)3537 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3538 {
3539 unsigned int seq;
3540 __u64 value;
3541
3542 if (unlikely(!vcpu->arch.cputm_enabled))
3543 return vcpu->arch.sie_block->cputm;
3544
3545 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3546 do {
3547 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3548 /*
3549 * If the writer would ever execute a read in the critical
3550 * section, e.g. in irq context, we have a deadlock.
3551 */
3552 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3553 value = vcpu->arch.sie_block->cputm;
3554 /* if cputm_start is 0, accounting is being started/stopped */
3555 if (likely(vcpu->arch.cputm_start))
3556 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3557 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3558 preempt_enable();
3559 return value;
3560 }
3561
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)3562 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3563 {
3564
3565 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
3566 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3567 __start_cpu_timer_accounting(vcpu);
3568 vcpu->cpu = cpu;
3569 }
3570
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)3571 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3572 {
3573 vcpu->cpu = -1;
3574 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3575 __stop_cpu_timer_accounting(vcpu);
3576 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
3577
3578 }
3579
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)3580 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
3581 {
3582 preempt_disable();
3583 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
3584 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
3585 preempt_enable();
3586
3587 if (!kvm_is_ucontrol(vcpu->kvm)) {
3588 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
3589 sca_add_vcpu(vcpu);
3590 }
3591 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 ||
3592 vcpu->kvm->arch.user_operexec)
3593 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3594
3595 /* Pairs with smp_load_acquire() in kvm_arch_vcpu_ioctl_run() and kvm_arch_vcpu_ioctl() */
3596 smp_store_release(&vcpu->arch.initialized, true);
3597 }
3598
kvm_has_pckmo_subfunc(struct kvm * kvm,unsigned long nr)3599 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3600 {
3601 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3602 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3603 return true;
3604 return false;
3605 }
3606
kvm_has_pckmo_ecc(struct kvm * kvm)3607 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3608 {
3609 /* At least one ECC subfunction must be present */
3610 return kvm_has_pckmo_subfunc(kvm, 32) ||
3611 kvm_has_pckmo_subfunc(kvm, 33) ||
3612 kvm_has_pckmo_subfunc(kvm, 34) ||
3613 kvm_has_pckmo_subfunc(kvm, 40) ||
3614 kvm_has_pckmo_subfunc(kvm, 41);
3615
3616 }
3617
kvm_has_pckmo_hmac(struct kvm * kvm)3618 static bool kvm_has_pckmo_hmac(struct kvm *kvm)
3619 {
3620 /* At least one HMAC subfunction must be present */
3621 return kvm_has_pckmo_subfunc(kvm, 118) ||
3622 kvm_has_pckmo_subfunc(kvm, 122);
3623 }
3624
kvm_s390_vcpu_crypto_setup(struct kvm_vcpu * vcpu)3625 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3626 {
3627 /*
3628 * If the AP instructions are not being interpreted and the MSAX3
3629 * facility is not configured for the guest, there is nothing to set up.
3630 */
3631 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
3632 return;
3633
3634 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
3635 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
3636 vcpu->arch.sie_block->eca &= ~ECA_APIE;
3637 vcpu->arch.sie_block->ecd &= ~(ECD_ECC | ECD_HMAC);
3638
3639 if (vcpu->kvm->arch.crypto.apie)
3640 vcpu->arch.sie_block->eca |= ECA_APIE;
3641
3642 /* Set up protected key support */
3643 if (vcpu->kvm->arch.crypto.aes_kw) {
3644 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
3645 /* ecc/hmac is also wrapped with AES key */
3646 if (kvm_has_pckmo_ecc(vcpu->kvm))
3647 vcpu->arch.sie_block->ecd |= ECD_ECC;
3648 if (kvm_has_pckmo_hmac(vcpu->kvm))
3649 vcpu->arch.sie_block->ecd |= ECD_HMAC;
3650 }
3651
3652 if (vcpu->kvm->arch.crypto.dea_kw)
3653 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
3654 }
3655
kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu * vcpu)3656 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3657 {
3658 if (vcpu->arch.sie_block->cbrlo)
3659 free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo));
3660 vcpu->arch.sie_block->cbrlo = 0;
3661 }
3662
kvm_s390_vcpu_setup_cmma(struct kvm_vcpu * vcpu)3663 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3664 {
3665 void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3666
3667 if (!cbrlo_page)
3668 return -ENOMEM;
3669
3670 vcpu->arch.sie_block->cbrlo = virt_to_phys(cbrlo_page);
3671 return 0;
3672 }
3673
kvm_s390_vcpu_setup_model(struct kvm_vcpu * vcpu)3674 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3675 {
3676 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3677
3678 vcpu->arch.sie_block->ibc = model->ibc;
3679 if (test_kvm_facility(vcpu->kvm, 7))
3680 vcpu->arch.sie_block->fac = virt_to_phys(model->fac_list);
3681 }
3682
kvm_s390_vcpu_setup(struct kvm_vcpu * vcpu)3683 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
3684 {
3685 int rc = 0;
3686 u16 uvrc, uvrrc;
3687
3688 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3689 CPUSTAT_SM |
3690 CPUSTAT_STOPPED);
3691
3692 if (test_kvm_facility(vcpu->kvm, 78))
3693 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
3694 else if (test_kvm_facility(vcpu->kvm, 8))
3695 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
3696
3697 kvm_s390_vcpu_setup_model(vcpu);
3698
3699 /* pgste_set_pte has special handling for !machine_has_esop() */
3700 if (machine_has_esop())
3701 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
3702 if (test_kvm_facility(vcpu->kvm, 9))
3703 vcpu->arch.sie_block->ecb |= ECB_SRSI;
3704 if (test_kvm_facility(vcpu->kvm, 11))
3705 vcpu->arch.sie_block->ecb |= ECB_PTF;
3706 if (test_kvm_facility(vcpu->kvm, 73))
3707 vcpu->arch.sie_block->ecb |= ECB_TE;
3708 if (!kvm_is_ucontrol(vcpu->kvm))
3709 vcpu->arch.sie_block->ecb |= ECB_SPECI;
3710
3711 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
3712 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
3713 if (test_kvm_facility(vcpu->kvm, 130))
3714 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3715 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3716 if (sclp.has_cei)
3717 vcpu->arch.sie_block->eca |= ECA_CEI;
3718 if (sclp.has_ib)
3719 vcpu->arch.sie_block->eca |= ECA_IB;
3720 if (sclp.has_siif)
3721 vcpu->arch.sie_block->eca |= ECA_SII;
3722 if (kvm_s390_use_sca_entries())
3723 vcpu->arch.sie_block->eca |= ECA_SIGPI;
3724 if (test_kvm_facility(vcpu->kvm, 129)) {
3725 vcpu->arch.sie_block->eca |= ECA_VX;
3726 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3727 }
3728 if (test_kvm_facility(vcpu->kvm, 139))
3729 vcpu->arch.sie_block->ecd |= ECD_MEF;
3730 if (test_kvm_facility(vcpu->kvm, 156))
3731 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3732 if (vcpu->arch.sie_block->gd) {
3733 vcpu->arch.sie_block->eca |= ECA_AIV;
3734 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3735 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3736 }
3737 vcpu->arch.sie_block->sdnxo = virt_to_phys(&vcpu->run->s.regs.sdnx) | SDNXC;
3738 vcpu->arch.sie_block->riccbd = virt_to_phys(&vcpu->run->s.regs.riccb);
3739
3740 if (sclp.has_kss)
3741 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3742 else
3743 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3744
3745 if (vcpu->kvm->arch.use_cmma) {
3746 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3747 if (rc)
3748 return rc;
3749 }
3750 hrtimer_setup(&vcpu->arch.ckc_timer, kvm_s390_idle_wakeup, CLOCK_MONOTONIC,
3751 HRTIMER_MODE_REL);
3752
3753 vcpu->arch.sie_block->hpid = HPID_KVM;
3754
3755 kvm_s390_vcpu_crypto_setup(vcpu);
3756
3757 kvm_s390_vcpu_pci_setup(vcpu);
3758
3759 if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3760 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3761 if (rc)
3762 kvm_s390_vcpu_unsetup_cmma(vcpu);
3763 }
3764
3765 return rc;
3766 }
3767
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)3768 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3769 {
3770 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3771 return -EINVAL;
3772 return 0;
3773 }
3774
DEFINE_FREE(sie_page,struct sie_page *,if (_T)free_page ((unsigned long)(_T)))3775 DEFINE_FREE(sie_page, struct sie_page *, if (_T) free_page((unsigned long)(_T)))
3776
3777 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
3778 {
3779 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
3780 struct sie_page *sie_page __free(sie_page) = NULL;
3781 int rc;
3782
3783 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3784 mc = kvm_s390_new_mmu_cache();
3785 if (!mc)
3786 return -ENOMEM;
3787 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
3788 if (!sie_page)
3789 return -ENOMEM;
3790
3791 vcpu->arch.sie_block = &sie_page->sie_block;
3792 vcpu->arch.sie_block->itdba = virt_to_phys(&sie_page->itdb);
3793
3794 /* the real guest size will always be smaller than msl */
3795 vcpu->arch.sie_block->mso = 0;
3796 vcpu->arch.sie_block->msl = sclp.hamax;
3797
3798 vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
3799 spin_lock_init(&vcpu->arch.local_int.lock);
3800 vcpu->arch.sie_block->gd = kvm_s390_get_gisa_desc(vcpu->kvm);
3801 seqcount_init(&vcpu->arch.cputm_seqcount);
3802
3803 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3804 kvm_clear_async_pf_completion_queue(vcpu);
3805 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3806 KVM_SYNC_GPRS |
3807 KVM_SYNC_ACRS |
3808 KVM_SYNC_CRS |
3809 KVM_SYNC_ARCH0 |
3810 KVM_SYNC_PFAULT |
3811 KVM_SYNC_DIAG318;
3812 vcpu->arch.acrs_loaded = false;
3813 kvm_s390_set_prefix(vcpu, 0);
3814 if (test_kvm_facility(vcpu->kvm, 64))
3815 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3816 if (test_kvm_facility(vcpu->kvm, 82))
3817 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3818 if (test_kvm_facility(vcpu->kvm, 133))
3819 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3820 if (test_kvm_facility(vcpu->kvm, 156))
3821 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3822 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3823 * cpu_has_vx(), (load|store)_fpu_regs() will work with vrs format.
3824 */
3825 if (cpu_has_vx())
3826 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3827 else
3828 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3829
3830 if (kvm_is_ucontrol(vcpu->kvm)) {
3831 vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL);
3832 if (!vcpu->arch.gmap)
3833 return -ENOMEM;
3834 }
3835
3836 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p",
3837 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3838 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3839
3840 rc = kvm_s390_vcpu_setup(vcpu);
3841 if (rc) {
3842 if (kvm_is_ucontrol(vcpu->kvm)) {
3843 scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
3844 gmap_remove_child(vcpu->arch.gmap);
3845 vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
3846 }
3847 return rc;
3848 }
3849
3850 vcpu->arch.mc = no_free_ptr(mc);
3851 sie_page = NULL;
3852 kvm_s390_update_topology_change_report(vcpu->kvm, 1);
3853 return 0;
3854 }
3855
kvm_arch_vcpu_runnable(struct kvm_vcpu * vcpu)3856 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3857 {
3858 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
3859 return kvm_s390_vcpu_has_irq(vcpu, 0);
3860 }
3861
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)3862 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3863 {
3864 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3865 }
3866
kvm_s390_vcpu_block(struct kvm_vcpu * vcpu)3867 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3868 {
3869 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3870 exit_sie(vcpu);
3871 }
3872
kvm_s390_vcpu_unblock(struct kvm_vcpu * vcpu)3873 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3874 {
3875 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3876 }
3877
kvm_s390_vcpu_request(struct kvm_vcpu * vcpu)3878 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3879 {
3880 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3881 exit_sie(vcpu);
3882 }
3883
kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu * vcpu)3884 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3885 {
3886 return atomic_read(&vcpu->arch.sie_block->prog20) &
3887 (PROG_BLOCK_SIE | PROG_REQUEST);
3888 }
3889
kvm_s390_vcpu_request_handled(struct kvm_vcpu * vcpu)3890 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3891 {
3892 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3893 }
3894
3895 /*
3896 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3897 * If the CPU is not running (e.g. waiting as idle) the function will
3898 * return immediately. */
exit_sie(struct kvm_vcpu * vcpu)3899 void exit_sie(struct kvm_vcpu *vcpu)
3900 {
3901 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3902 kvm_s390_vsie_kick(vcpu);
3903 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3904 cpu_relax();
3905 }
3906
3907 /* Kick a guest cpu out of SIE to process a request synchronously */
kvm_s390_sync_request(int req,struct kvm_vcpu * vcpu)3908 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3909 {
3910 __kvm_make_request(req, vcpu);
3911 kvm_s390_vcpu_request(vcpu);
3912 }
3913
kvm_arch_no_poll(struct kvm_vcpu * vcpu)3914 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3915 {
3916 /* do not poll with more than halt_poll_max_steal percent of steal time */
3917 if (get_lowcore()->avg_steal_timer * 100 / (TICK_USEC << 12) >=
3918 READ_ONCE(halt_poll_max_steal)) {
3919 vcpu->stat.halt_no_poll_steal++;
3920 return true;
3921 }
3922 return false;
3923 }
3924
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)3925 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3926 {
3927 /* kvm common code refers to this, but never calls it */
3928 BUG();
3929 return 0;
3930 }
3931
kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)3932 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3933 struct kvm_one_reg *reg)
3934 {
3935 int r = -EINVAL;
3936
3937 switch (reg->id) {
3938 case KVM_REG_S390_TODPR:
3939 r = put_user(vcpu->arch.sie_block->todpr,
3940 (u32 __user *)reg->addr);
3941 break;
3942 case KVM_REG_S390_EPOCHDIFF:
3943 r = put_user(vcpu->arch.sie_block->epoch,
3944 (u64 __user *)reg->addr);
3945 break;
3946 case KVM_REG_S390_CPU_TIMER:
3947 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3948 (u64 __user *)reg->addr);
3949 break;
3950 case KVM_REG_S390_CLOCK_COMP:
3951 r = put_user(vcpu->arch.sie_block->ckc,
3952 (u64 __user *)reg->addr);
3953 break;
3954 case KVM_REG_S390_PFTOKEN:
3955 r = put_user(vcpu->arch.pfault_token,
3956 (u64 __user *)reg->addr);
3957 break;
3958 case KVM_REG_S390_PFCOMPARE:
3959 r = put_user(vcpu->arch.pfault_compare,
3960 (u64 __user *)reg->addr);
3961 break;
3962 case KVM_REG_S390_PFSELECT:
3963 r = put_user(vcpu->arch.pfault_select,
3964 (u64 __user *)reg->addr);
3965 break;
3966 case KVM_REG_S390_PP:
3967 r = put_user(vcpu->arch.sie_block->pp,
3968 (u64 __user *)reg->addr);
3969 break;
3970 case KVM_REG_S390_GBEA:
3971 r = put_user(vcpu->arch.sie_block->gbea,
3972 (u64 __user *)reg->addr);
3973 break;
3974 default:
3975 break;
3976 }
3977
3978 return r;
3979 }
3980
kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)3981 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3982 struct kvm_one_reg *reg)
3983 {
3984 int r = -EINVAL;
3985 __u64 val;
3986
3987 switch (reg->id) {
3988 case KVM_REG_S390_TODPR:
3989 r = get_user(vcpu->arch.sie_block->todpr,
3990 (u32 __user *)reg->addr);
3991 break;
3992 case KVM_REG_S390_EPOCHDIFF:
3993 r = get_user(vcpu->arch.sie_block->epoch,
3994 (u64 __user *)reg->addr);
3995 break;
3996 case KVM_REG_S390_CPU_TIMER:
3997 r = get_user(val, (u64 __user *)reg->addr);
3998 if (!r)
3999 kvm_s390_set_cpu_timer(vcpu, val);
4000 break;
4001 case KVM_REG_S390_CLOCK_COMP:
4002 r = get_user(vcpu->arch.sie_block->ckc,
4003 (u64 __user *)reg->addr);
4004 break;
4005 case KVM_REG_S390_PFTOKEN:
4006 r = get_user(vcpu->arch.pfault_token,
4007 (u64 __user *)reg->addr);
4008 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4009 kvm_clear_async_pf_completion_queue(vcpu);
4010 break;
4011 case KVM_REG_S390_PFCOMPARE:
4012 r = get_user(vcpu->arch.pfault_compare,
4013 (u64 __user *)reg->addr);
4014 break;
4015 case KVM_REG_S390_PFSELECT:
4016 r = get_user(vcpu->arch.pfault_select,
4017 (u64 __user *)reg->addr);
4018 break;
4019 case KVM_REG_S390_PP:
4020 r = get_user(vcpu->arch.sie_block->pp,
4021 (u64 __user *)reg->addr);
4022 break;
4023 case KVM_REG_S390_GBEA:
4024 r = get_user(vcpu->arch.sie_block->gbea,
4025 (u64 __user *)reg->addr);
4026 break;
4027 default:
4028 break;
4029 }
4030
4031 return r;
4032 }
4033
kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu * vcpu)4034 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
4035 {
4036 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
4037 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
4038 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
4039
4040 kvm_clear_async_pf_completion_queue(vcpu);
4041 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
4042 kvm_s390_vcpu_stop(vcpu);
4043 kvm_s390_clear_local_irqs(vcpu);
4044 }
4045
kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu * vcpu)4046 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
4047 {
4048 /* Initial reset is a superset of the normal reset */
4049 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4050
4051 /*
4052 * This equals initial cpu reset in pop, but we don't switch to ESA.
4053 * We do not only reset the internal data, but also ...
4054 */
4055 vcpu->arch.sie_block->gpsw.mask = 0;
4056 vcpu->arch.sie_block->gpsw.addr = 0;
4057 kvm_s390_set_prefix(vcpu, 0);
4058 kvm_s390_set_cpu_timer(vcpu, 0);
4059 vcpu->arch.sie_block->ckc = 0;
4060 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
4061 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
4062 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
4063
4064 /* ... the data in sync regs */
4065 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
4066 vcpu->run->s.regs.ckc = 0;
4067 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
4068 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
4069 vcpu->run->psw_addr = 0;
4070 vcpu->run->psw_mask = 0;
4071 vcpu->run->s.regs.todpr = 0;
4072 vcpu->run->s.regs.cputm = 0;
4073 vcpu->run->s.regs.ckc = 0;
4074 vcpu->run->s.regs.pp = 0;
4075 vcpu->run->s.regs.gbea = 1;
4076 vcpu->run->s.regs.fpc = 0;
4077 /*
4078 * Do not reset these registers in the protected case, as some of
4079 * them are overlaid and they are not accessible in this case
4080 * anyway.
4081 */
4082 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4083 vcpu->arch.sie_block->gbea = 1;
4084 vcpu->arch.sie_block->pp = 0;
4085 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4086 vcpu->arch.sie_block->todpr = 0;
4087 }
4088 }
4089
kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu * vcpu)4090 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
4091 {
4092 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
4093
4094 /* Clear reset is a superset of the initial reset */
4095 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4096
4097 memset(®s->gprs, 0, sizeof(regs->gprs));
4098 memset(®s->vrs, 0, sizeof(regs->vrs));
4099 memset(®s->acrs, 0, sizeof(regs->acrs));
4100 memset(®s->gscb, 0, sizeof(regs->gscb));
4101
4102 regs->etoken = 0;
4103 regs->etoken_extension = 0;
4104 }
4105
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)4106 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4107 {
4108 vcpu_load(vcpu);
4109 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
4110 vcpu_put(vcpu);
4111 return 0;
4112 }
4113
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)4114 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4115 {
4116 vcpu_load(vcpu);
4117 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
4118 vcpu_put(vcpu);
4119 return 0;
4120 }
4121
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)4122 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
4123 struct kvm_sregs *sregs)
4124 {
4125 vcpu_load(vcpu);
4126
4127 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
4128 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
4129
4130 vcpu_put(vcpu);
4131 return 0;
4132 }
4133
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)4134 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
4135 struct kvm_sregs *sregs)
4136 {
4137 vcpu_load(vcpu);
4138
4139 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
4140 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
4141
4142 vcpu_put(vcpu);
4143 return 0;
4144 }
4145
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)4146 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4147 {
4148 vcpu_load(vcpu);
4149
4150 vcpu->run->s.regs.fpc = fpu->fpc;
4151 if (cpu_has_vx())
4152 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
4153 (freg_t *) fpu->fprs);
4154 else
4155 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4156
4157 vcpu_put(vcpu);
4158 return 0;
4159 }
4160
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)4161 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4162 {
4163 vcpu_load(vcpu);
4164
4165 if (cpu_has_vx())
4166 convert_vx_to_fp((freg_t *) fpu->fprs,
4167 (__vector128 *) vcpu->run->s.regs.vrs);
4168 else
4169 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
4170 fpu->fpc = vcpu->run->s.regs.fpc;
4171
4172 vcpu_put(vcpu);
4173 return 0;
4174 }
4175
kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu * vcpu,psw_t psw)4176 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
4177 {
4178 int rc = 0;
4179
4180 if (!is_vcpu_stopped(vcpu))
4181 rc = -EBUSY;
4182 else {
4183 vcpu->run->psw_mask = psw.mask;
4184 vcpu->run->psw_addr = psw.addr;
4185 }
4186 return rc;
4187 }
4188
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)4189 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
4190 struct kvm_translation *tr)
4191 {
4192 return -EINVAL; /* not implemented yet */
4193 }
4194
4195 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
4196 KVM_GUESTDBG_USE_HW_BP | \
4197 KVM_GUESTDBG_ENABLE)
4198
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)4199 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
4200 struct kvm_guest_debug *dbg)
4201 {
4202 int rc = 0;
4203
4204 vcpu_load(vcpu);
4205
4206 vcpu->guest_debug = 0;
4207 kvm_s390_clear_bp_data(vcpu);
4208
4209 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
4210 rc = -EINVAL;
4211 goto out;
4212 }
4213 if (!sclp.has_gpere) {
4214 rc = -EINVAL;
4215 goto out;
4216 }
4217
4218 if (dbg->control & KVM_GUESTDBG_ENABLE) {
4219 vcpu->guest_debug = dbg->control;
4220 /* enforce guest PER */
4221 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
4222
4223 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) {
4224 scoped_guard(srcu, &vcpu->kvm->srcu)
4225 rc = kvm_s390_import_bp_data(vcpu, dbg);
4226 }
4227 } else {
4228 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4229 vcpu->arch.guestdbg.last_bp = 0;
4230 }
4231
4232 if (rc) {
4233 vcpu->guest_debug = 0;
4234 kvm_s390_clear_bp_data(vcpu);
4235 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4236 }
4237
4238 out:
4239 vcpu_put(vcpu);
4240 return rc;
4241 }
4242
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)4243 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
4244 struct kvm_mp_state *mp_state)
4245 {
4246 int ret;
4247
4248 vcpu_load(vcpu);
4249
4250 /* CHECK_STOP and LOAD are not supported yet */
4251 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
4252 KVM_MP_STATE_OPERATING;
4253
4254 vcpu_put(vcpu);
4255 return ret;
4256 }
4257
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)4258 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
4259 struct kvm_mp_state *mp_state)
4260 {
4261 int rc = 0;
4262
4263 vcpu_load(vcpu);
4264
4265 /* user space knows about this interface - let it control the state */
4266 kvm_s390_set_user_cpu_state_ctrl(vcpu->kvm);
4267
4268 switch (mp_state->mp_state) {
4269 case KVM_MP_STATE_STOPPED:
4270 rc = kvm_s390_vcpu_stop(vcpu);
4271 break;
4272 case KVM_MP_STATE_OPERATING:
4273 rc = kvm_s390_vcpu_start(vcpu);
4274 break;
4275 case KVM_MP_STATE_LOAD:
4276 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4277 rc = -ENXIO;
4278 break;
4279 }
4280 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
4281 break;
4282 case KVM_MP_STATE_CHECK_STOP:
4283 fallthrough; /* CHECK_STOP and LOAD are not supported yet */
4284 default:
4285 rc = -ENXIO;
4286 }
4287
4288 vcpu_put(vcpu);
4289 return rc;
4290 }
4291
ibs_enabled(struct kvm_vcpu * vcpu)4292 static bool ibs_enabled(struct kvm_vcpu *vcpu)
4293 {
4294 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
4295 }
4296
vcpu_ucontrol_translate(struct kvm_vcpu * vcpu,gpa_t * gaddr)4297 static int vcpu_ucontrol_translate(struct kvm_vcpu *vcpu, gpa_t *gaddr)
4298 {
4299 int rc;
4300
4301 if (kvm_is_ucontrol(vcpu->kvm)) {
4302 rc = gmap_ucas_translate(vcpu->arch.mc, vcpu->arch.gmap, gaddr);
4303 if (rc == -EREMOTE) {
4304 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4305 vcpu->run->s390_ucontrol.trans_exc_code = *gaddr;
4306 vcpu->run->s390_ucontrol.pgm_code = PGM_SEGMENT_TRANSLATION;
4307 }
4308 return rc;
4309 }
4310 return 0;
4311 }
4312
kvm_s390_fixup_prefix(struct kvm_vcpu * vcpu)4313 static int kvm_s390_fixup_prefix(struct kvm_vcpu *vcpu)
4314 {
4315 gpa_t gaddr = kvm_s390_get_prefix(vcpu);
4316 gfn_t gfn;
4317 int rc;
4318
4319 if (vcpu_ucontrol_translate(vcpu, &gaddr))
4320 return -EREMOTE;
4321 gfn = gpa_to_gfn(gaddr);
4322
4323 rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gfn, true);
4324 if (rc)
4325 return rc;
4326 rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gfn + 1, true);
4327 if (rc)
4328 return rc;
4329
4330 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
4331 rc = dat_set_prefix_notif_bit(vcpu->kvm->arch.gmap->asce, gfn);
4332 return rc;
4333 }
4334
kvm_s390_handle_requests(struct kvm_vcpu * vcpu)4335 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
4336 {
4337 retry:
4338 kvm_s390_vcpu_request_handled(vcpu);
4339 if (!kvm_request_pending(vcpu))
4340 return 0;
4341 /*
4342 * If the guest prefix changed, re-arm the ipte notifier for the
4343 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
4344 * This ensures that the ipte instruction for this request has
4345 * already finished. We might race against a second unmapper that
4346 * wants to set the blocking bit. Lets just retry the request loop.
4347 */
4348 if (kvm_check_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu)) {
4349 int rc;
4350
4351 rc = kvm_s390_fixup_prefix(vcpu);
4352 if (rc) {
4353 kvm_make_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
4354 return rc;
4355 }
4356 goto retry;
4357 }
4358
4359 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
4360 vcpu->arch.sie_block->ihcpu = 0xffff;
4361 goto retry;
4362 }
4363
4364 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
4365 if (!ibs_enabled(vcpu)) {
4366 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
4367 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
4368 }
4369 goto retry;
4370 }
4371
4372 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
4373 if (ibs_enabled(vcpu)) {
4374 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
4375 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
4376 }
4377 goto retry;
4378 }
4379
4380 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
4381 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
4382 goto retry;
4383 }
4384
4385 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
4386 /*
4387 * Disable CMM virtualization; we will emulate the ESSA
4388 * instruction manually, in order to provide additional
4389 * functionalities needed for live migration.
4390 */
4391 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
4392 goto retry;
4393 }
4394
4395 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
4396 /*
4397 * Re-enable CMM virtualization if CMMA is available and
4398 * CMM has been used.
4399 */
4400 if (vcpu->kvm->arch.use_cmma && uses_cmm(vcpu->arch.gmap))
4401 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
4402 goto retry;
4403 }
4404
4405 /* we left the vsie handler, nothing to do, just clear the request */
4406 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
4407
4408 return 0;
4409 }
4410
__kvm_s390_set_tod_clock(struct kvm * kvm,const struct kvm_s390_vm_tod_clock * gtod)4411 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4412 {
4413 struct kvm_vcpu *vcpu;
4414 union tod_clock clk;
4415 unsigned long i;
4416
4417 preempt_disable();
4418
4419 store_tod_clock_ext(&clk);
4420
4421 kvm->arch.epoch = gtod->tod - clk.tod;
4422 kvm->arch.epdx = 0;
4423 if (test_kvm_facility(kvm, 139)) {
4424 kvm->arch.epdx = gtod->epoch_idx - clk.ei;
4425 if (kvm->arch.epoch > gtod->tod)
4426 kvm->arch.epdx -= 1;
4427 }
4428
4429 kvm_s390_vcpu_block_all(kvm);
4430 kvm_for_each_vcpu(i, vcpu, kvm) {
4431 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
4432 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
4433 }
4434
4435 kvm_s390_vcpu_unblock_all(kvm);
4436 preempt_enable();
4437 }
4438
kvm_s390_try_set_tod_clock(struct kvm * kvm,const struct kvm_s390_vm_tod_clock * gtod)4439 int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4440 {
4441 if (!mutex_trylock(&kvm->lock))
4442 return 0;
4443 __kvm_s390_set_tod_clock(kvm, gtod);
4444 mutex_unlock(&kvm->lock);
4445 return 1;
4446 }
4447
__kvm_inject_pfault_token(struct kvm_vcpu * vcpu,bool start_token,unsigned long token)4448 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
4449 unsigned long token)
4450 {
4451 struct kvm_s390_interrupt inti = {};
4452 struct kvm_s390_irq irq = {};
4453 struct kvm_s390_interrupt_info *inti_mem = NULL;
4454 int ret = 0;
4455
4456 if (start_token) {
4457 irq.u.ext.ext_params2 = token;
4458 irq.type = KVM_S390_INT_PFAULT_INIT;
4459 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
4460 } else {
4461 inti_mem = kzalloc_obj(*inti_mem, GFP_KERNEL_ACCOUNT);
4462 if (WARN_ON_ONCE(!inti_mem))
4463 return;
4464
4465 inti.type = KVM_S390_INT_PFAULT_DONE;
4466 inti.parm64 = token;
4467 ret = kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem);
4468 if (ret)
4469 kfree(inti_mem);
4470 WARN_ON_ONCE(ret);
4471 }
4472 }
4473
kvm_arch_async_page_not_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4474 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
4475 struct kvm_async_pf *work)
4476 {
4477 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
4478 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
4479
4480 return true;
4481 }
4482
kvm_arch_async_page_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4483 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
4484 struct kvm_async_pf *work)
4485 {
4486 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
4487 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
4488 }
4489
kvm_arch_async_page_ready(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4490 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
4491 struct kvm_async_pf *work)
4492 {
4493 /* s390 will always inject the page directly */
4494 }
4495
kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu * vcpu)4496 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
4497 {
4498 /*
4499 * s390 will always inject the page directly,
4500 * but we still want check_async_completion to cleanup
4501 */
4502 return true;
4503 }
4504
kvm_arch_setup_async_pf(struct kvm_vcpu * vcpu)4505 bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
4506 {
4507 hva_t hva;
4508 struct kvm_arch_async_pf arch;
4509
4510 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4511 return false;
4512 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
4513 vcpu->arch.pfault_compare)
4514 return false;
4515 if (psw_extint_disabled(vcpu))
4516 return false;
4517 if (kvm_s390_vcpu_has_irq(vcpu, 0))
4518 return false;
4519 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
4520 return false;
4521 if (!pfault_enabled(vcpu->arch.gmap))
4522 return false;
4523
4524 hva = gfn_to_hva(vcpu->kvm, current->thread.gmap_teid.addr);
4525 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
4526 return false;
4527
4528 return kvm_setup_async_pf(vcpu, current->thread.gmap_teid.addr * PAGE_SIZE, hva, &arch);
4529 }
4530
vcpu_pre_run(struct kvm_vcpu * vcpu)4531 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
4532 {
4533 int rc, cpuflags;
4534
4535 /*
4536 * On s390 notifications for arriving pages will be delivered directly
4537 * to the guest but the house keeping for completed pfaults is
4538 * handled outside the worker.
4539 */
4540 kvm_check_async_pf_completion(vcpu);
4541
4542 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
4543 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
4544
4545 if (!kvm_is_ucontrol(vcpu->kvm)) {
4546 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4547 if (rc || guestdbg_exit_pending(vcpu))
4548 return rc;
4549 }
4550
4551 rc = kvm_s390_handle_requests(vcpu);
4552 if (rc)
4553 return rc;
4554
4555 if (guestdbg_enabled(vcpu)) {
4556 kvm_s390_backup_guest_per_regs(vcpu);
4557 kvm_s390_patch_guest_per_regs(vcpu);
4558 }
4559
4560 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
4561
4562 vcpu->arch.sie_block->icptcode = 0;
4563 current->thread.gmap_int_code = 0;
4564 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4565 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4566 trace_kvm_s390_sie_enter(vcpu, cpuflags);
4567
4568 return 0;
4569 }
4570
vcpu_post_run_addressing_exception(struct kvm_vcpu * vcpu)4571 static int vcpu_post_run_addressing_exception(struct kvm_vcpu *vcpu)
4572 {
4573 struct kvm_s390_pgm_info pgm_info = {
4574 .code = PGM_ADDRESSING,
4575 };
4576 u8 opcode, ilen;
4577 int rc;
4578
4579 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4580 trace_kvm_s390_sie_fault(vcpu);
4581
4582 /*
4583 * We want to inject an addressing exception, which is defined as a
4584 * suppressing or terminating exception. However, since we came here
4585 * by a DAT access exception, the PSW still points to the faulting
4586 * instruction since DAT exceptions are nullifying. So we've got
4587 * to look up the current opcode to get the length of the instruction
4588 * to be able to forward the PSW.
4589 */
4590 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
4591 ilen = insn_length(opcode);
4592 if (rc < 0) {
4593 return rc;
4594 } else if (rc) {
4595 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4596 * Forward by arbitrary ilc, injection will take care of
4597 * nullification if necessary.
4598 */
4599 pgm_info = vcpu->arch.pgm;
4600 ilen = 4;
4601 }
4602 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4603 kvm_s390_forward_psw(vcpu, ilen);
4604 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
4605 }
4606
kvm_s390_assert_primary_as(struct kvm_vcpu * vcpu)4607 static void kvm_s390_assert_primary_as(struct kvm_vcpu *vcpu)
4608 {
4609 KVM_BUG(current->thread.gmap_teid.as != PSW_BITS_AS_PRIMARY, vcpu->kvm,
4610 "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4611 current->thread.gmap_int_code, current->thread.gmap_teid.val);
4612 }
4613
vcpu_dat_fault_handler(struct kvm_vcpu * vcpu,gpa_t gaddr,bool wr)4614 static int vcpu_dat_fault_handler(struct kvm_vcpu *vcpu, gpa_t gaddr, bool wr)
4615 {
4616 struct guest_fault f = {
4617 .write_attempt = wr,
4618 .attempt_pfault = pfault_enabled(vcpu->arch.gmap),
4619 };
4620 int rc;
4621
4622 if (vcpu_ucontrol_translate(vcpu, &gaddr))
4623 return -EREMOTE;
4624 f.gfn = gpa_to_gfn(gaddr);
4625
4626 rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
4627 if (rc <= 0)
4628 return rc;
4629 if (rc == PGM_ADDRESSING)
4630 return vcpu_post_run_addressing_exception(vcpu);
4631 KVM_BUG_ON(rc, vcpu->kvm);
4632 return -EINVAL;
4633 }
4634
vcpu_post_run_handle_fault(struct kvm_vcpu * vcpu)4635 static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu)
4636 {
4637 unsigned int foll = 0;
4638 unsigned long gaddr;
4639 int rc;
4640
4641 gaddr = current->thread.gmap_teid.addr * PAGE_SIZE;
4642 if (kvm_s390_cur_gmap_fault_is_write())
4643 foll = FOLL_WRITE;
4644
4645 switch (current->thread.gmap_int_code & PGM_INT_CODE_MASK) {
4646 case 0:
4647 vcpu->stat.exit_null++;
4648 break;
4649 case PGM_SECURE_STORAGE_ACCESS:
4650 case PGM_SECURE_STORAGE_VIOLATION:
4651 kvm_s390_assert_primary_as(vcpu);
4652 /*
4653 * This can happen after a reboot with asynchronous teardown;
4654 * the new guest (normal or protected) will run on top of the
4655 * previous protected guest. The old pages need to be destroyed
4656 * so the new guest can use them.
4657 */
4658 if (kvm_s390_pv_destroy_page(vcpu->kvm, gaddr)) {
4659 /*
4660 * Either KVM messed up the secure guest mapping or the
4661 * same page is mapped into multiple secure guests.
4662 *
4663 * This exception is only triggered when a guest 2 is
4664 * running and can therefore never occur in kernel
4665 * context.
4666 */
4667 pr_warn_ratelimited("Secure storage violation (%x) in task: %s, pid %d\n",
4668 current->thread.gmap_int_code, current->comm,
4669 current->pid);
4670 send_sig(SIGSEGV, current, 0);
4671 }
4672 break;
4673 case PGM_NON_SECURE_STORAGE_ACCESS:
4674 kvm_s390_assert_primary_as(vcpu);
4675 /*
4676 * This is normal operation; a page belonging to a protected
4677 * guest has not been imported yet. Try to import the page into
4678 * the protected guest.
4679 */
4680 rc = kvm_s390_pv_convert_to_secure(vcpu->kvm, gaddr);
4681 if (rc == -EINVAL)
4682 send_sig(SIGSEGV, current, 0);
4683 if (rc != -ENXIO)
4684 break;
4685 foll = FOLL_WRITE;
4686 fallthrough;
4687 case PGM_PROTECTION:
4688 case PGM_SEGMENT_TRANSLATION:
4689 case PGM_PAGE_TRANSLATION:
4690 case PGM_ASCE_TYPE:
4691 case PGM_REGION_FIRST_TRANS:
4692 case PGM_REGION_SECOND_TRANS:
4693 case PGM_REGION_THIRD_TRANS:
4694 kvm_s390_assert_primary_as(vcpu);
4695 return vcpu_dat_fault_handler(vcpu, gaddr, foll);
4696 default:
4697 KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4698 current->thread.gmap_int_code, current->thread.gmap_teid.val);
4699 send_sig(SIGSEGV, current, 0);
4700 break;
4701 }
4702 return 0;
4703 }
4704
vcpu_post_run(struct kvm_vcpu * vcpu,int sie_return)4705 static int vcpu_post_run(struct kvm_vcpu *vcpu, int sie_return)
4706 {
4707 struct mcck_volatile_info *mcck_info;
4708 struct sie_page *sie_page;
4709 int rc;
4710
4711 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4712 vcpu->arch.sie_block->icptcode);
4713 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4714
4715 if (guestdbg_enabled(vcpu))
4716 kvm_s390_restore_guest_per_regs(vcpu);
4717
4718 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4719 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
4720
4721 if (sie_return == SIE64_RETURN_MCCK) {
4722 sie_page = container_of(vcpu->arch.sie_block,
4723 struct sie_page, sie_block);
4724 mcck_info = &sie_page->mcck_info;
4725 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4726 return 0;
4727 }
4728 WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL);
4729
4730 if (vcpu->arch.sie_block->icptcode > 0) {
4731 rc = kvm_handle_sie_intercept(vcpu);
4732
4733 if (rc != -EOPNOTSUPP)
4734 return rc;
4735 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4736 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4737 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4738 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4739 return -EREMOTE;
4740 }
4741
4742 return vcpu_post_run_handle_fault(vcpu);
4743 }
4744
kvm_s390_enter_exit_sie(struct kvm_s390_sie_block * scb,u64 * gprs,unsigned long gasce)4745 int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
4746 u64 *gprs, unsigned long gasce)
4747 {
4748 int ret;
4749
4750 guest_state_enter_irqoff();
4751
4752 /*
4753 * The guest_state_{enter,exit}_irqoff() functions inform lockdep and
4754 * tracing that entry to the guest will enable host IRQs, and exit from
4755 * the guest will disable host IRQs.
4756 */
4757 ret = sie64a(scb, gprs, gasce);
4758
4759 guest_state_exit_irqoff();
4760
4761 return ret;
4762 }
4763
4764 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
__vcpu_run(struct kvm_vcpu * vcpu)4765 static int __vcpu_run(struct kvm_vcpu *vcpu)
4766 {
4767 int rc, sie_return;
4768 struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
4769
4770 /*
4771 * We try to hold kvm->srcu during most of vcpu_run (except when run-
4772 * ning the guest), so that memslots (and other stuff) are protected
4773 */
4774 kvm_vcpu_srcu_read_lock(vcpu);
4775
4776 while (true) {
4777 rc = vcpu_pre_run(vcpu);
4778 kvm_vcpu_srcu_read_unlock(vcpu);
4779 if (rc || guestdbg_exit_pending(vcpu))
4780 break;
4781
4782 /*
4783 * As PF_VCPU will be used in fault handler, between
4784 * guest_timing_enter_irqoff and guest_timing_exit_irqoff
4785 * should be no uaccess.
4786 */
4787 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4788 memcpy(sie_page->pv_grregs,
4789 vcpu->run->s.regs.gprs,
4790 sizeof(sie_page->pv_grregs));
4791 }
4792
4793 xfer_to_guest_mode_check:
4794 local_irq_disable();
4795 xfer_to_guest_mode_prepare();
4796 if (xfer_to_guest_mode_work_pending()) {
4797 local_irq_enable();
4798 rc = kvm_xfer_to_guest_mode_handle_work(vcpu);
4799 if (rc)
4800 break;
4801 goto xfer_to_guest_mode_check;
4802 }
4803
4804 guest_timing_enter_irqoff();
4805 __disable_cpu_timer_accounting(vcpu);
4806
4807 sie_return = kvm_s390_enter_exit_sie(vcpu->arch.sie_block,
4808 vcpu->run->s.regs.gprs,
4809 vcpu->arch.gmap->asce.val);
4810
4811 __enable_cpu_timer_accounting(vcpu);
4812 guest_timing_exit_irqoff();
4813 local_irq_enable();
4814
4815 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4816 memcpy(vcpu->run->s.regs.gprs,
4817 sie_page->pv_grregs,
4818 sizeof(sie_page->pv_grregs));
4819 /*
4820 * We're not allowed to inject interrupts on intercepts
4821 * that leave the guest state in an "in-between" state
4822 * where the next SIE entry will do a continuation.
4823 * Fence interrupts in our "internal" PSW.
4824 */
4825 if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
4826 vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
4827 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4828 }
4829 }
4830 kvm_vcpu_srcu_read_lock(vcpu);
4831
4832 rc = vcpu_post_run(vcpu, sie_return);
4833 if (rc || guestdbg_exit_pending(vcpu)) {
4834 kvm_vcpu_srcu_read_unlock(vcpu);
4835 break;
4836 }
4837 }
4838
4839 return rc;
4840 }
4841
sync_regs_fmt2(struct kvm_vcpu * vcpu)4842 static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
4843 {
4844 struct kvm_run *kvm_run = vcpu->run;
4845 struct runtime_instr_cb *riccb;
4846 struct gs_cb *gscb;
4847
4848 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4849 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
4850 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
4851 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
4852 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4853 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
4854 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
4855 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
4856 }
4857 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
4858 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
4859 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
4860 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
4861 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4862 kvm_clear_async_pf_completion_queue(vcpu);
4863 }
4864 if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
4865 vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
4866 vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
4867 VCPU_EVENT(vcpu, 3, "setting cpnc to %d", vcpu->arch.diag318_info.cpnc);
4868 }
4869 /*
4870 * If userspace sets the riccb (e.g. after migration) to a valid state,
4871 * we should enable RI here instead of doing the lazy enablement.
4872 */
4873 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4874 test_kvm_facility(vcpu->kvm, 64) &&
4875 riccb->v &&
4876 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4877 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
4878 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
4879 }
4880 /*
4881 * If userspace sets the gscb (e.g. after migration) to non-zero,
4882 * we should enable GS here instead of doing the lazy enablement.
4883 */
4884 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
4885 test_kvm_facility(vcpu->kvm, 133) &&
4886 gscb->gssm &&
4887 !vcpu->arch.gs_enabled) {
4888 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
4889 vcpu->arch.sie_block->ecb |= ECB_GS;
4890 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
4891 vcpu->arch.gs_enabled = 1;
4892 }
4893 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
4894 test_kvm_facility(vcpu->kvm, 82)) {
4895 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4896 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
4897 }
4898 if (cpu_has_gs()) {
4899 preempt_disable();
4900 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
4901 if (current->thread.gs_cb) {
4902 vcpu->arch.host_gscb = current->thread.gs_cb;
4903 save_gs_cb(vcpu->arch.host_gscb);
4904 }
4905 if (vcpu->arch.gs_enabled) {
4906 current->thread.gs_cb = (struct gs_cb *)
4907 &vcpu->run->s.regs.gscb;
4908 restore_gs_cb(current->thread.gs_cb);
4909 }
4910 preempt_enable();
4911 }
4912 /* SIE will load etoken directly from SDNX and therefore kvm_run */
4913 }
4914
sync_regs(struct kvm_vcpu * vcpu)4915 static void sync_regs(struct kvm_vcpu *vcpu)
4916 {
4917 struct kvm_run *kvm_run = vcpu->run;
4918
4919 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
4920 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
4921 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
4922 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
4923 /* some control register changes require a tlb flush */
4924 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4925 }
4926 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4927 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
4928 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
4929 }
4930 save_access_regs(vcpu->arch.host_acrs);
4931 restore_access_regs(vcpu->run->s.regs.acrs);
4932 vcpu->arch.acrs_loaded = true;
4933 kvm_s390_fpu_load(vcpu->run);
4934 /* Sync fmt2 only data */
4935 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
4936 sync_regs_fmt2(vcpu);
4937 } else {
4938 /*
4939 * In several places we have to modify our internal view to
4940 * not do things that are disallowed by the ultravisor. For
4941 * example we must not inject interrupts after specific exits
4942 * (e.g. 112 prefix page not secure). We do this by turning
4943 * off the machine check, external and I/O interrupt bits
4944 * of our PSW copy. To avoid getting validity intercepts, we
4945 * do only accept the condition code from userspace.
4946 */
4947 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
4948 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
4949 PSW_MASK_CC;
4950 }
4951
4952 kvm_run->kvm_dirty_regs = 0;
4953 }
4954
store_regs_fmt2(struct kvm_vcpu * vcpu)4955 static void store_regs_fmt2(struct kvm_vcpu *vcpu)
4956 {
4957 struct kvm_run *kvm_run = vcpu->run;
4958
4959 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
4960 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
4961 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
4962 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
4963 kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
4964 if (cpu_has_gs()) {
4965 preempt_disable();
4966 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
4967 if (vcpu->arch.gs_enabled)
4968 save_gs_cb(current->thread.gs_cb);
4969 current->thread.gs_cb = vcpu->arch.host_gscb;
4970 restore_gs_cb(vcpu->arch.host_gscb);
4971 if (!vcpu->arch.host_gscb)
4972 local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT);
4973 vcpu->arch.host_gscb = NULL;
4974 preempt_enable();
4975 }
4976 /* SIE will save etoken directly into SDNX and therefore kvm_run */
4977 }
4978
store_regs(struct kvm_vcpu * vcpu)4979 static void store_regs(struct kvm_vcpu *vcpu)
4980 {
4981 struct kvm_run *kvm_run = vcpu->run;
4982
4983 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
4984 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
4985 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
4986 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4987 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
4988 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
4989 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
4990 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
4991 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
4992 save_access_regs(vcpu->run->s.regs.acrs);
4993 restore_access_regs(vcpu->arch.host_acrs);
4994 vcpu->arch.acrs_loaded = false;
4995 kvm_s390_fpu_store(vcpu->run);
4996 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
4997 store_regs_fmt2(vcpu);
4998 }
4999
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)5000 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
5001 {
5002 struct kvm_run *kvm_run = vcpu->run;
5003 DECLARE_KERNEL_FPU_ONSTACK32(fpu);
5004 int rc;
5005
5006 /*
5007 * Running a VM while dumping always has the potential to
5008 * produce inconsistent dump data. But for PV vcpus a SIE
5009 * entry while dumping could also lead to a fatal validity
5010 * intercept which we absolutely want to avoid.
5011 */
5012 if (vcpu->kvm->arch.pv.dumping)
5013 return -EINVAL;
5014
5015 if (!vcpu->wants_to_run)
5016 return -EINTR;
5017
5018 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
5019 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
5020 return -EINVAL;
5021
5022 /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */
5023 if (!smp_load_acquire(&vcpu->arch.initialized))
5024 return -EINVAL;
5025
5026 vcpu_load(vcpu);
5027
5028 if (guestdbg_exit_pending(vcpu)) {
5029 kvm_s390_prepare_debug_exit(vcpu);
5030 rc = 0;
5031 goto out;
5032 }
5033
5034 kvm_sigset_activate(vcpu);
5035
5036 /*
5037 * no need to check the return value of vcpu_start as it can only have
5038 * an error for protvirt, but protvirt means user cpu state
5039 */
5040 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
5041 kvm_s390_vcpu_start(vcpu);
5042 } else if (is_vcpu_stopped(vcpu)) {
5043 pr_err_ratelimited("can't run stopped vcpu %d\n",
5044 vcpu->vcpu_id);
5045 rc = -EINVAL;
5046 goto out_sigset;
5047 }
5048
5049 kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR);
5050 sync_regs(vcpu);
5051 enable_cpu_timer_accounting(vcpu);
5052
5053 might_fault();
5054 rc = __vcpu_run(vcpu);
5055
5056 if (signal_pending(current) && !rc) {
5057 kvm_run->exit_reason = KVM_EXIT_INTR;
5058 vcpu->stat.signal_exits++;
5059 rc = -EINTR;
5060 }
5061
5062 if (guestdbg_exit_pending(vcpu) && !rc) {
5063 kvm_s390_prepare_debug_exit(vcpu);
5064 rc = 0;
5065 }
5066
5067 if (rc == -EREMOTE) {
5068 /* userspace support is needed, kvm_run has been prepared */
5069 rc = 0;
5070 }
5071
5072 disable_cpu_timer_accounting(vcpu);
5073 store_regs(vcpu);
5074 kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR);
5075
5076 vcpu->stat.exit_userspace++;
5077
5078 out_sigset:
5079 kvm_sigset_deactivate(vcpu);
5080
5081 out:
5082 vcpu_put(vcpu);
5083 return rc;
5084 }
5085
5086 /*
5087 * store status at address
5088 * we use have two special cases:
5089 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
5090 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
5091 */
kvm_s390_store_status_unloaded(struct kvm_vcpu * vcpu,unsigned long gpa)5092 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
5093 {
5094 unsigned char archmode = 1;
5095 freg_t fprs[NUM_FPRS];
5096 unsigned int px;
5097 u64 clkcomp, cputm;
5098 int rc;
5099
5100 px = kvm_s390_get_prefix(vcpu);
5101 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
5102 if (write_guest_abs(vcpu, 163, &archmode, 1))
5103 return -EFAULT;
5104 gpa = 0;
5105 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
5106 if (write_guest_real(vcpu, 163, &archmode, 1))
5107 return -EFAULT;
5108 gpa = px;
5109 } else
5110 gpa -= __LC_FPREGS_SAVE_AREA;
5111
5112 /* manually convert vector registers if necessary */
5113 if (cpu_has_vx()) {
5114 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
5115 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5116 fprs, 128);
5117 } else {
5118 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5119 vcpu->run->s.regs.fprs, 128);
5120 }
5121 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
5122 vcpu->run->s.regs.gprs, 128);
5123 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
5124 &vcpu->arch.sie_block->gpsw, 16);
5125 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
5126 &px, 4);
5127 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
5128 &vcpu->run->s.regs.fpc, 4);
5129 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
5130 &vcpu->arch.sie_block->todpr, 4);
5131 cputm = kvm_s390_get_cpu_timer(vcpu);
5132 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
5133 &cputm, 8);
5134 clkcomp = vcpu->arch.sie_block->ckc >> 8;
5135 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
5136 &clkcomp, 8);
5137 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
5138 &vcpu->run->s.regs.acrs, 64);
5139 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
5140 &vcpu->arch.sie_block->gcr, 128);
5141 return rc ? -EFAULT : 0;
5142 }
5143
kvm_s390_vcpu_store_status(struct kvm_vcpu * vcpu,unsigned long addr)5144 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
5145 {
5146 /*
5147 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
5148 * switch in the run ioctl. Let's update our copies before we save
5149 * it into the save area
5150 */
5151 kvm_s390_fpu_store(vcpu->run);
5152 save_access_regs(vcpu->run->s.regs.acrs);
5153
5154 return kvm_s390_store_status_unloaded(vcpu, addr);
5155 }
5156
__disable_ibs_on_vcpu(struct kvm_vcpu * vcpu)5157 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5158 {
5159 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
5160 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
5161 }
5162
__disable_ibs_on_all_vcpus(struct kvm * kvm)5163 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
5164 {
5165 unsigned long i;
5166 struct kvm_vcpu *vcpu;
5167
5168 kvm_for_each_vcpu(i, vcpu, kvm) {
5169 __disable_ibs_on_vcpu(vcpu);
5170 }
5171 }
5172
__enable_ibs_on_vcpu(struct kvm_vcpu * vcpu)5173 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5174 {
5175 if (!sclp.has_ibs)
5176 return;
5177 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
5178 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
5179 }
5180
kvm_s390_vcpu_start(struct kvm_vcpu * vcpu)5181 int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
5182 {
5183 int i, online_vcpus, r = 0, started_vcpus = 0;
5184
5185 if (!is_vcpu_stopped(vcpu))
5186 return 0;
5187
5188 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
5189 /* Only one cpu at a time may enter/leave the STOPPED state. */
5190 spin_lock(&vcpu->kvm->arch.start_stop_lock);
5191 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
5192
5193 /* Let's tell the UV that we want to change into the operating state */
5194 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5195 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
5196 if (r) {
5197 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5198 return r;
5199 }
5200 }
5201
5202 for (i = 0; i < online_vcpus; i++) {
5203 if (!is_vcpu_stopped(kvm_get_vcpu(vcpu->kvm, i)))
5204 started_vcpus++;
5205 }
5206
5207 if (started_vcpus == 0) {
5208 /* we're the only active VCPU -> speed it up */
5209 __enable_ibs_on_vcpu(vcpu);
5210 } else if (started_vcpus == 1) {
5211 /*
5212 * As we are starting a second VCPU, we have to disable
5213 * the IBS facility on all VCPUs to remove potentially
5214 * outstanding ENABLE requests.
5215 */
5216 __disable_ibs_on_all_vcpus(vcpu->kvm);
5217 }
5218
5219 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
5220 /*
5221 * The real PSW might have changed due to a RESTART interpreted by the
5222 * ultravisor. We block all interrupts and let the next sie exit
5223 * refresh our view.
5224 */
5225 if (kvm_s390_pv_cpu_is_protected(vcpu))
5226 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
5227 /*
5228 * Another VCPU might have used IBS while we were offline.
5229 * Let's play safe and flush the VCPU at startup.
5230 */
5231 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
5232 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5233 return 0;
5234 }
5235
kvm_s390_vcpu_stop(struct kvm_vcpu * vcpu)5236 int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
5237 {
5238 int i, online_vcpus, r = 0, started_vcpus = 0;
5239 struct kvm_vcpu *started_vcpu = NULL;
5240
5241 if (is_vcpu_stopped(vcpu))
5242 return 0;
5243
5244 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
5245 /* Only one cpu at a time may enter/leave the STOPPED state. */
5246 spin_lock(&vcpu->kvm->arch.start_stop_lock);
5247 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
5248
5249 /* Let's tell the UV that we want to change into the stopped state */
5250 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5251 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
5252 if (r) {
5253 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5254 return r;
5255 }
5256 }
5257
5258 /*
5259 * Set the VCPU to STOPPED and THEN clear the interrupt flag,
5260 * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
5261 * have been fully processed. This will ensure that the VCPU
5262 * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
5263 */
5264 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
5265 kvm_s390_clear_stop_irq(vcpu);
5266
5267 __disable_ibs_on_vcpu(vcpu);
5268
5269 for (i = 0; i < online_vcpus; i++) {
5270 struct kvm_vcpu *tmp = kvm_get_vcpu(vcpu->kvm, i);
5271
5272 if (!is_vcpu_stopped(tmp)) {
5273 started_vcpus++;
5274 started_vcpu = tmp;
5275 }
5276 }
5277
5278 if (started_vcpus == 1) {
5279 /*
5280 * As we only have one VCPU left, we want to enable the
5281 * IBS facility for that VCPU to speed it up.
5282 */
5283 __enable_ibs_on_vcpu(started_vcpu);
5284 }
5285
5286 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5287 return 0;
5288 }
5289
kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu * vcpu,struct kvm_enable_cap * cap)5290 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5291 struct kvm_enable_cap *cap)
5292 {
5293 int r;
5294
5295 if (cap->flags)
5296 return -EINVAL;
5297
5298 switch (cap->cap) {
5299 case KVM_CAP_S390_CSS_SUPPORT:
5300 if (!vcpu->kvm->arch.css_support) {
5301 vcpu->kvm->arch.css_support = 1;
5302 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
5303 trace_kvm_s390_enable_css(vcpu->kvm);
5304 }
5305 r = 0;
5306 break;
5307 default:
5308 r = -EINVAL;
5309 break;
5310 }
5311 return r;
5312 }
5313
kvm_s390_vcpu_sida_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5314 static long kvm_s390_vcpu_sida_op(struct kvm_vcpu *vcpu,
5315 struct kvm_s390_mem_op *mop)
5316 {
5317 void __user *uaddr = (void __user *)mop->buf;
5318 void *sida_addr;
5319 int r = 0;
5320
5321 if (mop->flags || !mop->size)
5322 return -EINVAL;
5323 if (mop->size + mop->sida_offset < mop->size)
5324 return -EINVAL;
5325 if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
5326 return -E2BIG;
5327 if (!kvm_s390_pv_cpu_is_protected(vcpu))
5328 return -EINVAL;
5329
5330 sida_addr = (char *)sida_addr(vcpu->arch.sie_block) + mop->sida_offset;
5331
5332 switch (mop->op) {
5333 case KVM_S390_MEMOP_SIDA_READ:
5334 if (copy_to_user(uaddr, sida_addr, mop->size))
5335 r = -EFAULT;
5336
5337 break;
5338 case KVM_S390_MEMOP_SIDA_WRITE:
5339 if (copy_from_user(sida_addr, uaddr, mop->size))
5340 r = -EFAULT;
5341 break;
5342 }
5343 return r;
5344 }
5345
kvm_s390_vcpu_mem_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5346 static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu,
5347 struct kvm_s390_mem_op *mop)
5348 {
5349 void __user *uaddr = (void __user *)mop->buf;
5350 void *tmpbuf __free(kvfree) = NULL;
5351 enum gacc_mode acc_mode;
5352 int r;
5353
5354 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_INJECT_EXCEPTION |
5355 KVM_S390_MEMOP_F_CHECK_ONLY |
5356 KVM_S390_MEMOP_F_SKEY_PROTECTION);
5357 if (r)
5358 return r;
5359 if (mop->ar >= NUM_ACRS)
5360 return -EINVAL;
5361 if (kvm_s390_pv_cpu_is_protected(vcpu))
5362 return -EINVAL;
5363 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
5364 tmpbuf = vmalloc(mop->size);
5365 if (!tmpbuf)
5366 return -ENOMEM;
5367 }
5368
5369 acc_mode = mop->op == KVM_S390_MEMOP_LOGICAL_READ ? GACC_FETCH : GACC_STORE;
5370 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
5371 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size,
5372 acc_mode, mop->key);
5373 } else if (acc_mode == GACC_FETCH) {
5374 r = read_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
5375 mop->size, mop->key);
5376 if (!r && copy_to_user(uaddr, tmpbuf, mop->size))
5377 return -EFAULT;
5378 } else {
5379 if (copy_from_user(tmpbuf, uaddr, mop->size))
5380 return -EFAULT;
5381 r = write_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
5382 mop->size, mop->key);
5383 }
5384
5385 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
5386 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
5387
5388 return r;
5389 }
5390
kvm_s390_vcpu_memsida_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5391 static long kvm_s390_vcpu_memsida_op(struct kvm_vcpu *vcpu,
5392 struct kvm_s390_mem_op *mop)
5393 {
5394 int r, srcu_idx;
5395
5396 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
5397
5398 switch (mop->op) {
5399 case KVM_S390_MEMOP_LOGICAL_READ:
5400 case KVM_S390_MEMOP_LOGICAL_WRITE:
5401 r = kvm_s390_vcpu_mem_op(vcpu, mop);
5402 break;
5403 case KVM_S390_MEMOP_SIDA_READ:
5404 case KVM_S390_MEMOP_SIDA_WRITE:
5405 /* we are locked against sida going away by the vcpu->mutex */
5406 r = kvm_s390_vcpu_sida_op(vcpu, mop);
5407 break;
5408 default:
5409 r = -EINVAL;
5410 }
5411
5412 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
5413 return r;
5414 }
5415
kvm_arch_vcpu_unlocked_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5416 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
5417 unsigned long arg)
5418 {
5419 struct kvm_vcpu *vcpu = filp->private_data;
5420 void __user *argp = (void __user *)arg;
5421 int rc;
5422
5423 switch (ioctl) {
5424 case KVM_S390_IRQ: {
5425 struct kvm_s390_irq s390irq;
5426
5427 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
5428 return -EFAULT;
5429 scoped_guard(srcu, &vcpu->kvm->srcu)
5430 rc = kvm_s390_inject_vcpu(vcpu, &s390irq);
5431 break;
5432 }
5433 case KVM_S390_INTERRUPT: {
5434 struct kvm_s390_interrupt s390int;
5435 struct kvm_s390_irq s390irq = {};
5436
5437 if (kvm_is_ucontrol(vcpu->kvm))
5438 return -EINVAL;
5439 if (copy_from_user(&s390int, argp, sizeof(s390int)))
5440 return -EFAULT;
5441 if (s390int_to_s390irq(&s390int, &s390irq))
5442 return -EINVAL;
5443 scoped_guard(srcu, &vcpu->kvm->srcu)
5444 rc = kvm_s390_inject_vcpu(vcpu, &s390irq);
5445 break;
5446 }
5447 default:
5448 rc = -ENOIOCTLCMD;
5449 break;
5450 }
5451
5452 /*
5453 * To simplify single stepping of userspace-emulated instructions,
5454 * KVM_EXIT_S390_SIEIC exit sets KVM_GUESTDBG_EXIT_PENDING (see
5455 * should_handle_per_ifetch()). However, if userspace emulation injects
5456 * an interrupt, it needs to be cleared, so that KVM_EXIT_DEBUG happens
5457 * after (and not before) the interrupt delivery.
5458 */
5459 if (!rc)
5460 vcpu->guest_debug &= ~KVM_GUESTDBG_EXIT_PENDING;
5461
5462 return rc;
5463 }
5464
kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu * vcpu,struct kvm_pv_cmd * cmd)5465 static int kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu *vcpu,
5466 struct kvm_pv_cmd *cmd)
5467 {
5468 struct kvm_s390_pv_dmp dmp;
5469 void *data;
5470 int ret;
5471
5472 /* Dump initialization is a prerequisite */
5473 if (!vcpu->kvm->arch.pv.dumping)
5474 return -EINVAL;
5475
5476 if (copy_from_user(&dmp, (__u8 __user *)cmd->data, sizeof(dmp)))
5477 return -EFAULT;
5478
5479 /* We only handle this subcmd right now */
5480 if (dmp.subcmd != KVM_PV_DUMP_CPU)
5481 return -EINVAL;
5482
5483 /* CPU dump length is the same as create cpu storage donation. */
5484 if (dmp.buff_len != uv_info.guest_cpu_stor_len)
5485 return -EINVAL;
5486
5487 data = kvzalloc(uv_info.guest_cpu_stor_len, GFP_KERNEL);
5488 if (!data)
5489 return -ENOMEM;
5490
5491 ret = kvm_s390_pv_dump_cpu(vcpu, data, &cmd->rc, &cmd->rrc);
5492
5493 VCPU_EVENT(vcpu, 3, "PROTVIRT DUMP CPU %d rc %x rrc %x",
5494 vcpu->vcpu_id, cmd->rc, cmd->rrc);
5495
5496 if (ret)
5497 ret = -EINVAL;
5498
5499 /* On success copy over the dump data */
5500 if (!ret && copy_to_user((__u8 __user *)dmp.buff_addr, data, uv_info.guest_cpu_stor_len))
5501 ret = -EFAULT;
5502
5503 kvfree(data);
5504 return ret;
5505 }
5506
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5507 long kvm_arch_vcpu_ioctl(struct file *filp,
5508 unsigned int ioctl, unsigned long arg)
5509 {
5510 struct kvm_vcpu *vcpu = filp->private_data;
5511 void __user *argp = (void __user *)arg;
5512 int idx;
5513 long r;
5514 u16 rc, rrc;
5515
5516 /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */
5517 if (!smp_load_acquire(&vcpu->arch.initialized))
5518 return -EINVAL;
5519
5520 vcpu_load(vcpu);
5521
5522 switch (ioctl) {
5523 case KVM_S390_STORE_STATUS:
5524 idx = srcu_read_lock(&vcpu->kvm->srcu);
5525 r = kvm_s390_store_status_unloaded(vcpu, arg);
5526 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5527 break;
5528 case KVM_S390_SET_INITIAL_PSW: {
5529 psw_t psw;
5530
5531 r = -EFAULT;
5532 if (copy_from_user(&psw, argp, sizeof(psw)))
5533 break;
5534 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
5535 break;
5536 }
5537 case KVM_S390_CLEAR_RESET:
5538 r = 0;
5539 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
5540 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5541 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5542 UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
5543 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
5544 rc, rrc);
5545 }
5546 break;
5547 case KVM_S390_INITIAL_RESET:
5548 r = 0;
5549 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
5550 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5551 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5552 UVC_CMD_CPU_RESET_INITIAL,
5553 &rc, &rrc);
5554 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
5555 rc, rrc);
5556 }
5557 break;
5558 case KVM_S390_NORMAL_RESET:
5559 r = 0;
5560 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
5561 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5562 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5563 UVC_CMD_CPU_RESET, &rc, &rrc);
5564 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
5565 rc, rrc);
5566 }
5567 break;
5568 case KVM_SET_ONE_REG:
5569 case KVM_GET_ONE_REG: {
5570 struct kvm_one_reg reg;
5571 r = -EINVAL;
5572 if (kvm_s390_pv_cpu_is_protected(vcpu))
5573 break;
5574 r = -EFAULT;
5575 if (copy_from_user(®, argp, sizeof(reg)))
5576 break;
5577 if (ioctl == KVM_SET_ONE_REG)
5578 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
5579 else
5580 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
5581 break;
5582 }
5583 #ifdef CONFIG_KVM_S390_UCONTROL
5584 case KVM_S390_UCAS_MAP: {
5585 struct kvm_s390_ucas_mapping ucas;
5586
5587 r = -EFAULT;
5588 if (copy_from_user(&ucas, argp, sizeof(ucas)))
5589 break;
5590
5591 r = -EINVAL;
5592 if (!kvm_is_ucontrol(vcpu->kvm))
5593 break;
5594 if (!IS_ALIGNED(ucas.user_addr | ucas.vcpu_addr | ucas.length, _SEGMENT_SIZE))
5595 break;
5596
5597 r = gmap_ucas_map(vcpu->arch.gmap, gpa_to_gfn(ucas.user_addr),
5598 gpa_to_gfn(ucas.vcpu_addr),
5599 ucas.length >> _SEGMENT_SHIFT);
5600 break;
5601 }
5602 case KVM_S390_UCAS_UNMAP: {
5603 struct kvm_s390_ucas_mapping ucas;
5604
5605 r = -EFAULT;
5606 if (copy_from_user(&ucas, argp, sizeof(ucas)))
5607 break;
5608
5609 r = -EINVAL;
5610 if (!kvm_is_ucontrol(vcpu->kvm))
5611 break;
5612 if (!IS_ALIGNED(ucas.vcpu_addr | ucas.length, _SEGMENT_SIZE))
5613 break;
5614
5615 gmap_ucas_unmap(vcpu->arch.gmap, gpa_to_gfn(ucas.vcpu_addr),
5616 ucas.length >> _SEGMENT_SHIFT);
5617 r = 0;
5618 break;
5619 }
5620 #endif
5621 case KVM_S390_VCPU_FAULT: {
5622 gpa_t gaddr = arg;
5623
5624 scoped_guard(srcu, &vcpu->kvm->srcu) {
5625 r = vcpu_ucontrol_translate(vcpu, &gaddr);
5626 if (r)
5627 break;
5628
5629 r = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(gaddr), false);
5630 if (r == PGM_ADDRESSING)
5631 r = -EFAULT;
5632 if (r <= 0)
5633 break;
5634 r = -EIO;
5635 KVM_BUG_ON(r, vcpu->kvm);
5636 }
5637 break;
5638 }
5639 case KVM_ENABLE_CAP:
5640 {
5641 struct kvm_enable_cap cap;
5642 r = -EFAULT;
5643 if (copy_from_user(&cap, argp, sizeof(cap)))
5644 break;
5645 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5646 break;
5647 }
5648 case KVM_S390_MEM_OP: {
5649 struct kvm_s390_mem_op mem_op;
5650
5651 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
5652 r = kvm_s390_vcpu_memsida_op(vcpu, &mem_op);
5653 else
5654 r = -EFAULT;
5655 break;
5656 }
5657 case KVM_S390_SET_IRQ_STATE: {
5658 struct kvm_s390_irq_state irq_state;
5659
5660 r = -EFAULT;
5661 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5662 break;
5663 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
5664 irq_state.len == 0 ||
5665 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
5666 r = -EINVAL;
5667 break;
5668 }
5669 /* do not use irq_state.flags, it will break old QEMUs */
5670 r = kvm_s390_set_irq_state(vcpu,
5671 (void __user *) irq_state.buf,
5672 irq_state.len);
5673 break;
5674 }
5675 case KVM_S390_GET_IRQ_STATE: {
5676 struct kvm_s390_irq_state irq_state;
5677
5678 r = -EFAULT;
5679 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5680 break;
5681 if (irq_state.len == 0) {
5682 r = -EINVAL;
5683 break;
5684 }
5685 /* do not use irq_state.flags, it will break old QEMUs */
5686 r = kvm_s390_get_irq_state(vcpu,
5687 (__u8 __user *) irq_state.buf,
5688 irq_state.len);
5689 break;
5690 }
5691 case KVM_S390_PV_CPU_COMMAND: {
5692 struct kvm_pv_cmd cmd;
5693
5694 r = -EINVAL;
5695 if (!is_prot_virt_host())
5696 break;
5697
5698 r = -EFAULT;
5699 if (copy_from_user(&cmd, argp, sizeof(cmd)))
5700 break;
5701
5702 r = -EINVAL;
5703 if (cmd.flags)
5704 break;
5705
5706 /* We only handle this cmd right now */
5707 if (cmd.cmd != KVM_PV_DUMP)
5708 break;
5709
5710 r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd);
5711
5712 /* Always copy over UV rc / rrc data */
5713 if (copy_to_user(argp + offsetof(struct kvm_pv_cmd, rc), &cmd.rc,
5714 sizeof(cmd.rc) + sizeof(cmd.rrc)))
5715 r = -EFAULT;
5716 break;
5717 }
5718 default:
5719 r = -ENOTTY;
5720 }
5721
5722 vcpu_put(vcpu);
5723 return r;
5724 }
5725
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)5726 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5727 {
5728 #ifdef CONFIG_KVM_S390_UCONTROL
5729 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
5730 && (kvm_is_ucontrol(vcpu->kvm))) {
5731 vmf->page = virt_to_page(vcpu->arch.sie_block);
5732 get_page(vmf->page);
5733 return 0;
5734 }
5735 #endif
5736 return VM_FAULT_SIGBUS;
5737 }
5738
kvm_arch_irqchip_in_kernel(struct kvm * kvm)5739 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
5740 {
5741 return true;
5742 }
5743
5744 /* Section: memory related */
kvm_arch_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)5745 int kvm_arch_prepare_memory_region(struct kvm *kvm,
5746 const struct kvm_memory_slot *old,
5747 struct kvm_memory_slot *new,
5748 enum kvm_mr_change change)
5749 {
5750 return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change);
5751 }
5752
cmma_d_count_pte(union pte * ptep,gfn_t gfn,gfn_t next,struct dat_walk * walk)5753 static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
5754 {
5755 union pgste pgste;
5756
5757 pgste = pgste_get_lock(ptep);
5758 if (pgste.cmma_d) {
5759 pgste.cmma_d = 0;
5760 atomic64_dec(walk->priv);
5761 }
5762 pgste_set_unlock(ptep, pgste);
5763 return 0;
5764 }
5765
kvm_s390_update_cmma_dirty(struct kvm * kvm,const struct kvm_memory_slot * old)5766 void kvm_s390_update_cmma_dirty(struct kvm *kvm, const struct kvm_memory_slot *old)
5767 {
5768 const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, };
5769
5770 if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) {
5771 _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages,
5772 kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES,
5773 &kvm->arch.cmma_dirty_pages);
5774 }
5775 }
5776
kvm_arch_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)5777 void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old,
5778 const struct kvm_memory_slot *new,
5779 enum kvm_mr_change change)
5780 {
5781 }
5782
5783 /**
5784 * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables
5785 * @vcpu: the vcpu that shall appear to have generated the fault-in.
5786 * @range: the range that needs to be faulted in.
5787 *
5788 * The first page of the given range is faulted in and the corresponding gmap
5789 * page tables are created, as if the given vCPU had performed a read
5790 * operation.
5791 * If the range starts outside any memslots, an error is returned. An error is
5792 * also returned for UCONTROL VMs, which should instead use the
5793 * KVM_S390_VCPU_FAULT ioctl.
5794 *
5795 * Return:
5796 * * %-ENOENT if the range lies outside of a memslot.
5797 * * %-EINVAL in case of invalid state (for example if the VM is UCONTROL).
5798 * * %-EIO if errors happen while faulting-in the page (will trigger a warning
5799 * in the caller).
5800 * * other error codes < 0 in case of other errors.
5801 * * otherwise a number > 0 of bytes that have been faulted in successfully.
5802 */
kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu * vcpu,struct kvm_pre_fault_memory * range)5803 long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range)
5804 {
5805 struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), };
5806 gpa_t end;
5807 int rc;
5808
5809 if (kvm_is_ucontrol(vcpu->kvm))
5810 return -EINVAL;
5811
5812 rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
5813 if (rc == PGM_ADDRESSING)
5814 return -ENOENT;
5815 if (rc > 0)
5816 return -EIO;
5817 if (rc < 0)
5818 return rc;
5819
5820 if (f.ptep)
5821 return PAGE_SIZE;
5822
5823 end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE);
5824 return min(range->size, end - range->gpa);
5825 }
5826
5827 /**
5828 * kvm_test_age_gfn() - test young
5829 * @kvm: the kvm instance
5830 * @range: the range of guest addresses whose young status needs to be cleared
5831 *
5832 * Context: called by KVM common code without holding the kvm mmu lock
5833 * Return: true if any page in the given range is young, otherwise 0.
5834 */
kvm_test_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)5835 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
5836 {
5837 scoped_guard(read_lock, &kvm->mmu_lock)
5838 return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end);
5839 }
5840
5841 /**
5842 * kvm_age_gfn() - clear young
5843 * @kvm: the kvm instance
5844 * @range: the range of guest addresses whose young status needs to be cleared
5845 *
5846 * Context: called by KVM common code without holding the kvm mmu lock
5847 * Return: true if any page in the given range was young, otherwise 0.
5848 */
kvm_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)5849 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
5850 {
5851 scoped_guard(read_lock, &kvm->mmu_lock)
5852 return gmap_age_gfn(kvm->arch.gmap, range->start, range->end);
5853 }
5854
5855 /**
5856 * kvm_unmap_gfn_range() - Unmap a range of guest addresses
5857 * @kvm: the kvm instance
5858 * @range: the range of guest page frames to invalidate
5859 *
5860 * This function always returns false because every DAT table modification
5861 * has to use the appropriate DAT table manipulation instructions, which will
5862 * keep the TLB coherent, hence no additional TLB flush is ever required.
5863 *
5864 * Context: called by KVM common code with the kvm mmu write lock held
5865 * Return: false
5866 */
kvm_unmap_gfn_range(struct kvm * kvm,struct kvm_gfn_range * range)5867 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
5868 {
5869 return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end);
5870 }
5871
nonhyp_mask(int i)5872 static inline unsigned long nonhyp_mask(int i)
5873 {
5874 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
5875
5876 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
5877 }
5878
kvm_s390_init(void)5879 static int __init kvm_s390_init(void)
5880 {
5881 int i, r;
5882
5883 if (!sclp.has_sief2) {
5884 pr_info("SIE is not available\n");
5885 return -ENODEV;
5886 }
5887
5888 if (hpage_2g && !hpage) {
5889 hpage_2g = 0;
5890 pr_info("Disabling 2G hugepage support, since 1M hugepage support is not enabled.\n");
5891 }
5892
5893 for (i = 0; i < HMFAI_DWORDS; i++)
5894 kvm_s390_fac_base[i] |= nonhyp_mask(i);
5895
5896 r = __kvm_s390_init();
5897 if (r)
5898 return r;
5899
5900 r = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
5901 if (r) {
5902 __kvm_s390_exit();
5903 return r;
5904 }
5905 return 0;
5906 }
5907
kvm_s390_exit(void)5908 static void __exit kvm_s390_exit(void)
5909 {
5910 kvm_exit();
5911
5912 __kvm_s390_exit();
5913 }
5914
5915 module_init(kvm_s390_init);
5916 module_exit(kvm_s390_exit);
5917
5918 /*
5919 * Enable autoloading of the kvm module.
5920 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
5921 * since x86 takes a different approach.
5922 */
5923 #include <linux/miscdevice.h>
5924 MODULE_ALIAS_MISCDEV(KVM_MINOR);
5925 MODULE_ALIAS("devname:kvm");
5926