1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * hosting IBM Z kernel virtual machines (s390x)
4 *
5 * Copyright IBM Corp. 2008, 2020
6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Christian Ehrhardt <ehrhardt@de.ibm.com>
10 * Jason J. Herne <jjherne@us.ibm.com>
11 */
12
13 #define pr_fmt(fmt) "kvm-s390: " fmt
14
15 #include <linux/compiler.h>
16 #include <linux/entry-virt.h>
17 #include <linux/export.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/cpufeature.h>
28 #include <linux/random.h>
29 #include <linux/slab.h>
30 #include <linux/timer.h>
31 #include <linux/vmalloc.h>
32 #include <linux/bitmap.h>
33 #include <linux/sched/signal.h>
34 #include <linux/string.h>
35 #include <linux/pgtable.h>
36 #include <linux/mmu_notifier.h>
37
38 #include <asm/access-regs.h>
39 #include <asm/asm-offsets.h>
40 #include <asm/lowcore.h>
41 #include <asm/machine.h>
42 #include <asm/stp.h>
43 #include <asm/gmap.h>
44 #include <asm/gmap_helpers.h>
45 #include <asm/nmi.h>
46 #include <asm/isc.h>
47 #include <asm/sclp.h>
48 #include <asm/cpacf.h>
49 #include <asm/timex.h>
50 #include <asm/asm.h>
51 #include <asm/fpu.h>
52 #include <asm/ap.h>
53 #include <asm/uv.h>
54 #include "kvm-s390.h"
55 #include "gaccess.h"
56 #include "pci.h"
57
58 #define CREATE_TRACE_POINTS
59 #include "trace.h"
60 #include "trace-s390.h"
61
62 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
63 #define LOCAL_IRQS 32
64 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
65 (KVM_MAX_VCPUS + LOCAL_IRQS))
66
67 const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
68 KVM_GENERIC_VM_STATS(),
69 STATS_DESC_COUNTER(VM, inject_io),
70 STATS_DESC_COUNTER(VM, inject_float_mchk),
71 STATS_DESC_COUNTER(VM, inject_pfault_done),
72 STATS_DESC_COUNTER(VM, inject_service_signal),
73 STATS_DESC_COUNTER(VM, inject_virtio),
74 STATS_DESC_COUNTER(VM, aen_forward),
75 STATS_DESC_COUNTER(VM, gmap_shadow_reuse),
76 STATS_DESC_COUNTER(VM, gmap_shadow_create),
77 STATS_DESC_COUNTER(VM, gmap_shadow_r1_entry),
78 STATS_DESC_COUNTER(VM, gmap_shadow_r2_entry),
79 STATS_DESC_COUNTER(VM, gmap_shadow_r3_entry),
80 STATS_DESC_COUNTER(VM, gmap_shadow_sg_entry),
81 STATS_DESC_COUNTER(VM, gmap_shadow_pg_entry),
82 };
83
84 const struct kvm_stats_header kvm_vm_stats_header = {
85 .name_size = KVM_STATS_NAME_SIZE,
86 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
87 .id_offset = sizeof(struct kvm_stats_header),
88 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
89 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
90 sizeof(kvm_vm_stats_desc),
91 };
92
93 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
94 KVM_GENERIC_VCPU_STATS(),
95 STATS_DESC_COUNTER(VCPU, exit_userspace),
96 STATS_DESC_COUNTER(VCPU, exit_null),
97 STATS_DESC_COUNTER(VCPU, exit_external_request),
98 STATS_DESC_COUNTER(VCPU, exit_io_request),
99 STATS_DESC_COUNTER(VCPU, exit_external_interrupt),
100 STATS_DESC_COUNTER(VCPU, exit_stop_request),
101 STATS_DESC_COUNTER(VCPU, exit_validity),
102 STATS_DESC_COUNTER(VCPU, exit_instruction),
103 STATS_DESC_COUNTER(VCPU, exit_pei),
104 STATS_DESC_COUNTER(VCPU, halt_no_poll_steal),
105 STATS_DESC_COUNTER(VCPU, instruction_lctl),
106 STATS_DESC_COUNTER(VCPU, instruction_lctlg),
107 STATS_DESC_COUNTER(VCPU, instruction_stctl),
108 STATS_DESC_COUNTER(VCPU, instruction_stctg),
109 STATS_DESC_COUNTER(VCPU, exit_program_interruption),
110 STATS_DESC_COUNTER(VCPU, exit_instr_and_program),
111 STATS_DESC_COUNTER(VCPU, exit_operation_exception),
112 STATS_DESC_COUNTER(VCPU, deliver_ckc),
113 STATS_DESC_COUNTER(VCPU, deliver_cputm),
114 STATS_DESC_COUNTER(VCPU, deliver_external_call),
115 STATS_DESC_COUNTER(VCPU, deliver_emergency_signal),
116 STATS_DESC_COUNTER(VCPU, deliver_service_signal),
117 STATS_DESC_COUNTER(VCPU, deliver_virtio),
118 STATS_DESC_COUNTER(VCPU, deliver_stop_signal),
119 STATS_DESC_COUNTER(VCPU, deliver_prefix_signal),
120 STATS_DESC_COUNTER(VCPU, deliver_restart_signal),
121 STATS_DESC_COUNTER(VCPU, deliver_program),
122 STATS_DESC_COUNTER(VCPU, deliver_io),
123 STATS_DESC_COUNTER(VCPU, deliver_machine_check),
124 STATS_DESC_COUNTER(VCPU, exit_wait_state),
125 STATS_DESC_COUNTER(VCPU, inject_ckc),
126 STATS_DESC_COUNTER(VCPU, inject_cputm),
127 STATS_DESC_COUNTER(VCPU, inject_external_call),
128 STATS_DESC_COUNTER(VCPU, inject_emergency_signal),
129 STATS_DESC_COUNTER(VCPU, inject_mchk),
130 STATS_DESC_COUNTER(VCPU, inject_pfault_init),
131 STATS_DESC_COUNTER(VCPU, inject_program),
132 STATS_DESC_COUNTER(VCPU, inject_restart),
133 STATS_DESC_COUNTER(VCPU, inject_set_prefix),
134 STATS_DESC_COUNTER(VCPU, inject_stop_signal),
135 STATS_DESC_COUNTER(VCPU, instruction_epsw),
136 STATS_DESC_COUNTER(VCPU, instruction_gs),
137 STATS_DESC_COUNTER(VCPU, instruction_io_other),
138 STATS_DESC_COUNTER(VCPU, instruction_lpsw),
139 STATS_DESC_COUNTER(VCPU, instruction_lpswe),
140 STATS_DESC_COUNTER(VCPU, instruction_lpswey),
141 STATS_DESC_COUNTER(VCPU, instruction_pfmf),
142 STATS_DESC_COUNTER(VCPU, instruction_ptff),
143 STATS_DESC_COUNTER(VCPU, instruction_sck),
144 STATS_DESC_COUNTER(VCPU, instruction_sckpf),
145 STATS_DESC_COUNTER(VCPU, instruction_stidp),
146 STATS_DESC_COUNTER(VCPU, instruction_spx),
147 STATS_DESC_COUNTER(VCPU, instruction_stpx),
148 STATS_DESC_COUNTER(VCPU, instruction_stap),
149 STATS_DESC_COUNTER(VCPU, instruction_iske),
150 STATS_DESC_COUNTER(VCPU, instruction_ri),
151 STATS_DESC_COUNTER(VCPU, instruction_rrbe),
152 STATS_DESC_COUNTER(VCPU, instruction_sske),
153 STATS_DESC_COUNTER(VCPU, instruction_ipte_interlock),
154 STATS_DESC_COUNTER(VCPU, instruction_stsi),
155 STATS_DESC_COUNTER(VCPU, instruction_stfl),
156 STATS_DESC_COUNTER(VCPU, instruction_tb),
157 STATS_DESC_COUNTER(VCPU, instruction_tpi),
158 STATS_DESC_COUNTER(VCPU, instruction_tprot),
159 STATS_DESC_COUNTER(VCPU, instruction_tsch),
160 STATS_DESC_COUNTER(VCPU, instruction_sie),
161 STATS_DESC_COUNTER(VCPU, instruction_essa),
162 STATS_DESC_COUNTER(VCPU, instruction_sthyi),
163 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense),
164 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense_running),
165 STATS_DESC_COUNTER(VCPU, instruction_sigp_external_call),
166 STATS_DESC_COUNTER(VCPU, instruction_sigp_emergency),
167 STATS_DESC_COUNTER(VCPU, instruction_sigp_cond_emergency),
168 STATS_DESC_COUNTER(VCPU, instruction_sigp_start),
169 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop),
170 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop_store_status),
171 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_status),
172 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_adtl_status),
173 STATS_DESC_COUNTER(VCPU, instruction_sigp_arch),
174 STATS_DESC_COUNTER(VCPU, instruction_sigp_prefix),
175 STATS_DESC_COUNTER(VCPU, instruction_sigp_restart),
176 STATS_DESC_COUNTER(VCPU, instruction_sigp_init_cpu_reset),
177 STATS_DESC_COUNTER(VCPU, instruction_sigp_cpu_reset),
178 STATS_DESC_COUNTER(VCPU, instruction_sigp_unknown),
179 STATS_DESC_COUNTER(VCPU, instruction_diagnose_10),
180 STATS_DESC_COUNTER(VCPU, instruction_diagnose_44),
181 STATS_DESC_COUNTER(VCPU, instruction_diagnose_9c),
182 STATS_DESC_COUNTER(VCPU, diag_9c_ignored),
183 STATS_DESC_COUNTER(VCPU, diag_9c_forward),
184 STATS_DESC_COUNTER(VCPU, instruction_diagnose_258),
185 STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),
186 STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),
187 STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),
188 STATS_DESC_COUNTER(VCPU, pfault_sync),
189 STATS_DESC_COUNTER(VCPU, signal_exits)
190 };
191
192 const struct kvm_stats_header kvm_vcpu_stats_header = {
193 .name_size = KVM_STATS_NAME_SIZE,
194 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
195 .id_offset = sizeof(struct kvm_stats_header),
196 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
197 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
198 sizeof(kvm_vcpu_stats_desc),
199 };
200
201 /* allow nested virtualization in KVM (if enabled by user space) */
202 static int nested;
203 module_param(nested, int, S_IRUGO);
204 MODULE_PARM_DESC(nested, "Nested virtualization support");
205
206 /* allow 1m huge page guest backing, if !nested */
207 static int hpage;
208 module_param(hpage, int, 0444);
209 MODULE_PARM_DESC(hpage, "1m huge page backing support");
210
211 /* maximum percentage of steal time for polling. >100 is treated like 100 */
212 static u8 halt_poll_max_steal = 10;
213 module_param(halt_poll_max_steal, byte, 0644);
214 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
215
216 /* if set to true, the GISA will be initialized and used if available */
217 static bool use_gisa = true;
218 module_param(use_gisa, bool, 0644);
219 MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
220
221 /* maximum diag9c forwarding per second */
222 unsigned int diag9c_forwarding_hz;
223 module_param(diag9c_forwarding_hz, uint, 0644);
224 MODULE_PARM_DESC(diag9c_forwarding_hz, "Maximum diag9c forwarding per second, 0 to turn off");
225
226 /*
227 * allow asynchronous deinit for protected guests; enable by default since
228 * the feature is opt-in anyway
229 */
230 static int async_destroy = 1;
231 module_param(async_destroy, int, 0444);
232 MODULE_PARM_DESC(async_destroy, "Asynchronous destroy for protected guests");
233
234 /*
235 * For now we handle at most 16 double words as this is what the s390 base
236 * kernel handles and stores in the prefix page. If we ever need to go beyond
237 * this, this requires changes to code, but the external uapi can stay.
238 */
239 #define SIZE_INTERNAL 16
240
241 /*
242 * Base feature mask that defines default mask for facilities. Consists of the
243 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
244 */
245 static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
246 /*
247 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
248 * and defines the facilities that can be enabled via a cpu model.
249 */
250 static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
251
kvm_s390_fac_size(void)252 static unsigned long kvm_s390_fac_size(void)
253 {
254 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
255 BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
256 BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
257 sizeof(stfle_fac_list));
258
259 return SIZE_INTERNAL;
260 }
261
262 /* available cpu features supported by kvm */
263 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
264 /* available subfunctions indicated via query / "test bit" */
265 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
266
267 static struct gmap_notifier gmap_notifier;
268 static struct gmap_notifier vsie_gmap_notifier;
269 debug_info_t *kvm_s390_dbf;
270 debug_info_t *kvm_s390_dbf_uv;
271
272 /* Section: not file related */
273 /* forward declarations */
274 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
275 unsigned long end);
276
kvm_clock_sync_scb(struct kvm_s390_sie_block * scb,u64 delta)277 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
278 {
279 u8 delta_idx = 0;
280
281 /*
282 * The TOD jumps by delta, we have to compensate this by adding
283 * -delta to the epoch.
284 */
285 delta = -delta;
286
287 /* sign-extension - we're adding to signed values below */
288 if ((s64)delta < 0)
289 delta_idx = -1;
290
291 scb->epoch += delta;
292 if (scb->ecd & ECD_MEF) {
293 scb->epdx += delta_idx;
294 if (scb->epoch < delta)
295 scb->epdx += 1;
296 }
297 }
298
299 /*
300 * This callback is executed during stop_machine(). All CPUs are therefore
301 * temporarily stopped. In order not to change guest behavior, we have to
302 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
303 * so a CPU won't be stopped while calculating with the epoch.
304 */
kvm_clock_sync(struct notifier_block * notifier,unsigned long val,void * v)305 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
306 void *v)
307 {
308 struct kvm *kvm;
309 struct kvm_vcpu *vcpu;
310 unsigned long i;
311 unsigned long long *delta = v;
312
313 list_for_each_entry(kvm, &vm_list, vm_list) {
314 kvm_for_each_vcpu(i, vcpu, kvm) {
315 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
316 if (i == 0) {
317 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
318 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
319 }
320 if (vcpu->arch.cputm_enabled)
321 vcpu->arch.cputm_start += *delta;
322 if (vcpu->arch.vsie_block)
323 kvm_clock_sync_scb(vcpu->arch.vsie_block,
324 *delta);
325 }
326 }
327 return NOTIFY_OK;
328 }
329
330 static struct notifier_block kvm_clock_notifier = {
331 .notifier_call = kvm_clock_sync,
332 };
333
allow_cpu_feat(unsigned long nr)334 static void allow_cpu_feat(unsigned long nr)
335 {
336 set_bit_inv(nr, kvm_s390_available_cpu_feat);
337 }
338
plo_test_bit(unsigned char nr)339 static inline int plo_test_bit(unsigned char nr)
340 {
341 unsigned long function = (unsigned long)nr | 0x100;
342 int cc;
343
344 asm volatile(
345 " lgr 0,%[function]\n"
346 /* Parameter registers are ignored for "test bit" */
347 " plo 0,0,0,0(0)\n"
348 CC_IPM(cc)
349 : CC_OUT(cc, cc)
350 : [function] "d" (function)
351 : CC_CLOBBER_LIST("0"));
352 return CC_TRANSFORM(cc) == 0;
353 }
354
pfcr_query(u8 (* query)[16])355 static __always_inline void pfcr_query(u8 (*query)[16])
356 {
357 asm volatile(
358 " lghi 0,0\n"
359 " .insn rsy,0xeb0000000016,0,0,%[query]"
360 : [query] "=QS" (*query)
361 :
362 : "cc", "0");
363 }
364
__sortl_query(u8 (* query)[32])365 static __always_inline void __sortl_query(u8 (*query)[32])
366 {
367 asm volatile(
368 " lghi 0,0\n"
369 " la 1,%[query]\n"
370 /* Parameter registers are ignored */
371 " .insn rre,0xb9380000,2,4"
372 : [query] "=R" (*query)
373 :
374 : "cc", "0", "1");
375 }
376
__dfltcc_query(u8 (* query)[32])377 static __always_inline void __dfltcc_query(u8 (*query)[32])
378 {
379 asm volatile(
380 " lghi 0,0\n"
381 " la 1,%[query]\n"
382 /* Parameter registers are ignored */
383 " .insn rrf,0xb9390000,2,4,6,0"
384 : [query] "=R" (*query)
385 :
386 : "cc", "0", "1");
387 }
388
kvm_s390_cpu_feat_init(void)389 static void __init kvm_s390_cpu_feat_init(void)
390 {
391 int i;
392
393 for (i = 0; i < 256; ++i) {
394 if (plo_test_bit(i))
395 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
396 }
397
398 if (test_facility(28)) /* TOD-clock steering */
399 ptff(kvm_s390_available_subfunc.ptff,
400 sizeof(kvm_s390_available_subfunc.ptff),
401 PTFF_QAF);
402
403 if (test_facility(17)) { /* MSA */
404 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
405 kvm_s390_available_subfunc.kmac);
406 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
407 kvm_s390_available_subfunc.kmc);
408 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
409 kvm_s390_available_subfunc.km);
410 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
411 kvm_s390_available_subfunc.kimd);
412 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
413 kvm_s390_available_subfunc.klmd);
414 }
415 if (test_facility(76)) /* MSA3 */
416 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
417 kvm_s390_available_subfunc.pckmo);
418 if (test_facility(77)) { /* MSA4 */
419 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
420 kvm_s390_available_subfunc.kmctr);
421 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
422 kvm_s390_available_subfunc.kmf);
423 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
424 kvm_s390_available_subfunc.kmo);
425 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
426 kvm_s390_available_subfunc.pcc);
427 }
428 if (test_facility(57)) /* MSA5 */
429 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
430 kvm_s390_available_subfunc.ppno);
431
432 if (test_facility(146)) /* MSA8 */
433 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
434 kvm_s390_available_subfunc.kma);
435
436 if (test_facility(155)) /* MSA9 */
437 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
438 kvm_s390_available_subfunc.kdsa);
439
440 if (test_facility(150)) /* SORTL */
441 __sortl_query(&kvm_s390_available_subfunc.sortl);
442
443 if (test_facility(151)) /* DFLTCC */
444 __dfltcc_query(&kvm_s390_available_subfunc.dfltcc);
445
446 if (test_facility(201)) /* PFCR */
447 pfcr_query(&kvm_s390_available_subfunc.pfcr);
448
449 if (machine_has_esop())
450 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
451 /*
452 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
453 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
454 */
455 if (!sclp.has_sief2 || !machine_has_esop() || !sclp.has_64bscao ||
456 !test_facility(3) || !nested)
457 return;
458 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
459 if (sclp.has_64bscao)
460 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
461 if (sclp.has_siif)
462 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
463 if (sclp.has_gpere)
464 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
465 if (sclp.has_gsls)
466 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
467 if (sclp.has_ib)
468 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
469 if (sclp.has_cei)
470 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
471 if (sclp.has_ibs)
472 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
473 if (sclp.has_kss)
474 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
475 /*
476 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
477 * all skey handling functions read/set the skey from the PGSTE
478 * instead of the real storage key.
479 *
480 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
481 * pages being detected as preserved although they are resident.
482 *
483 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
484 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
485 *
486 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
487 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
488 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
489 *
490 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
491 * cannot easily shadow the SCA because of the ipte lock.
492 */
493 }
494
__kvm_s390_init(void)495 static int __init __kvm_s390_init(void)
496 {
497 int rc = -ENOMEM;
498
499 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
500 if (!kvm_s390_dbf)
501 return -ENOMEM;
502
503 kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
504 if (!kvm_s390_dbf_uv)
505 goto err_kvm_uv;
506
507 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
508 debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
509 goto err_debug_view;
510
511 kvm_s390_cpu_feat_init();
512
513 /* Register floating interrupt controller interface. */
514 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
515 if (rc) {
516 pr_err("A FLIC registration call failed with rc=%d\n", rc);
517 goto err_flic;
518 }
519
520 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
521 rc = kvm_s390_pci_init();
522 if (rc) {
523 pr_err("Unable to allocate AIFT for PCI\n");
524 goto err_pci;
525 }
526 }
527
528 rc = kvm_s390_gib_init(GAL_ISC);
529 if (rc)
530 goto err_gib;
531
532 gmap_notifier.notifier_call = kvm_gmap_notifier;
533 gmap_register_pte_notifier(&gmap_notifier);
534 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
535 gmap_register_pte_notifier(&vsie_gmap_notifier);
536 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
537 &kvm_clock_notifier);
538
539 return 0;
540
541 err_gib:
542 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
543 kvm_s390_pci_exit();
544 err_pci:
545 err_flic:
546 err_debug_view:
547 debug_unregister(kvm_s390_dbf_uv);
548 err_kvm_uv:
549 debug_unregister(kvm_s390_dbf);
550 return rc;
551 }
552
__kvm_s390_exit(void)553 static void __kvm_s390_exit(void)
554 {
555 gmap_unregister_pte_notifier(&gmap_notifier);
556 gmap_unregister_pte_notifier(&vsie_gmap_notifier);
557 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
558 &kvm_clock_notifier);
559
560 kvm_s390_gib_destroy();
561 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
562 kvm_s390_pci_exit();
563 debug_unregister(kvm_s390_dbf);
564 debug_unregister(kvm_s390_dbf_uv);
565 }
566
567 /* Section: device related */
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)568 long kvm_arch_dev_ioctl(struct file *filp,
569 unsigned int ioctl, unsigned long arg)
570 {
571 if (ioctl == KVM_S390_ENABLE_SIE)
572 return s390_enable_sie();
573 return -EINVAL;
574 }
575
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)576 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
577 {
578 int r;
579
580 switch (ext) {
581 case KVM_CAP_S390_PSW:
582 case KVM_CAP_S390_GMAP:
583 case KVM_CAP_SYNC_MMU:
584 #ifdef CONFIG_KVM_S390_UCONTROL
585 case KVM_CAP_S390_UCONTROL:
586 #endif
587 case KVM_CAP_ASYNC_PF:
588 case KVM_CAP_SYNC_REGS:
589 case KVM_CAP_ONE_REG:
590 case KVM_CAP_ENABLE_CAP:
591 case KVM_CAP_S390_CSS_SUPPORT:
592 case KVM_CAP_IOEVENTFD:
593 case KVM_CAP_S390_IRQCHIP:
594 case KVM_CAP_VM_ATTRIBUTES:
595 case KVM_CAP_MP_STATE:
596 case KVM_CAP_IMMEDIATE_EXIT:
597 case KVM_CAP_S390_INJECT_IRQ:
598 case KVM_CAP_S390_USER_SIGP:
599 case KVM_CAP_S390_USER_STSI:
600 case KVM_CAP_S390_SKEYS:
601 case KVM_CAP_S390_IRQ_STATE:
602 case KVM_CAP_S390_USER_INSTR0:
603 case KVM_CAP_S390_CMMA_MIGRATION:
604 case KVM_CAP_S390_AIS:
605 case KVM_CAP_S390_AIS_MIGRATION:
606 case KVM_CAP_S390_VCPU_RESETS:
607 case KVM_CAP_SET_GUEST_DEBUG:
608 case KVM_CAP_S390_DIAG318:
609 case KVM_CAP_IRQFD_RESAMPLE:
610 case KVM_CAP_S390_USER_OPEREXEC:
611 r = 1;
612 break;
613 case KVM_CAP_SET_GUEST_DEBUG2:
614 r = KVM_GUESTDBG_VALID_MASK;
615 break;
616 case KVM_CAP_S390_HPAGE_1M:
617 r = 0;
618 if (hpage && !(kvm && kvm_is_ucontrol(kvm)))
619 r = 1;
620 break;
621 case KVM_CAP_S390_MEM_OP:
622 r = MEM_OP_MAX_SIZE;
623 break;
624 case KVM_CAP_S390_MEM_OP_EXTENSION:
625 /*
626 * Flag bits indicating which extensions are supported.
627 * If r > 0, the base extension must also be supported/indicated,
628 * in order to maintain backwards compatibility.
629 */
630 r = KVM_S390_MEMOP_EXTENSION_CAP_BASE |
631 KVM_S390_MEMOP_EXTENSION_CAP_CMPXCHG;
632 break;
633 case KVM_CAP_NR_VCPUS:
634 case KVM_CAP_MAX_VCPUS:
635 case KVM_CAP_MAX_VCPU_ID:
636 /*
637 * Return the same value for KVM_CAP_MAX_VCPUS and
638 * KVM_CAP_MAX_VCPU_ID to conform with the KVM API.
639 */
640 r = KVM_S390_ESCA_CPU_SLOTS;
641 if (!kvm_s390_use_sca_entries())
642 r = KVM_MAX_VCPUS;
643 if (ext == KVM_CAP_NR_VCPUS)
644 r = min_t(unsigned int, num_online_cpus(), r);
645 break;
646 case KVM_CAP_S390_COW:
647 r = machine_has_esop();
648 break;
649 case KVM_CAP_S390_VECTOR_REGISTERS:
650 r = test_facility(129);
651 break;
652 case KVM_CAP_S390_RI:
653 r = test_facility(64);
654 break;
655 case KVM_CAP_S390_GS:
656 r = test_facility(133);
657 break;
658 case KVM_CAP_S390_BPB:
659 r = test_facility(82);
660 break;
661 case KVM_CAP_S390_PROTECTED_ASYNC_DISABLE:
662 r = async_destroy && is_prot_virt_host();
663 break;
664 case KVM_CAP_S390_PROTECTED:
665 r = is_prot_virt_host();
666 break;
667 case KVM_CAP_S390_PROTECTED_DUMP: {
668 u64 pv_cmds_dump[] = {
669 BIT_UVC_CMD_DUMP_INIT,
670 BIT_UVC_CMD_DUMP_CONFIG_STOR_STATE,
671 BIT_UVC_CMD_DUMP_CPU,
672 BIT_UVC_CMD_DUMP_COMPLETE,
673 };
674 int i;
675
676 r = is_prot_virt_host();
677
678 for (i = 0; i < ARRAY_SIZE(pv_cmds_dump); i++) {
679 if (!test_bit_inv(pv_cmds_dump[i],
680 (unsigned long *)&uv_info.inst_calls_list)) {
681 r = 0;
682 break;
683 }
684 }
685 break;
686 }
687 case KVM_CAP_S390_ZPCI_OP:
688 r = kvm_s390_pci_interp_allowed();
689 break;
690 case KVM_CAP_S390_CPU_TOPOLOGY:
691 r = test_facility(11);
692 break;
693 default:
694 r = 0;
695 }
696 return r;
697 }
698
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)699 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
700 {
701 int i;
702 gfn_t cur_gfn, last_gfn;
703 unsigned long gaddr, vmaddr;
704 struct gmap *gmap = kvm->arch.gmap;
705 DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
706
707 /* Loop over all guest segments */
708 cur_gfn = memslot->base_gfn;
709 last_gfn = memslot->base_gfn + memslot->npages;
710 for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
711 gaddr = gfn_to_gpa(cur_gfn);
712 vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
713 if (kvm_is_error_hva(vmaddr))
714 continue;
715
716 bitmap_zero(bitmap, _PAGE_ENTRIES);
717 gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
718 for (i = 0; i < _PAGE_ENTRIES; i++) {
719 if (test_bit(i, bitmap))
720 mark_page_dirty(kvm, cur_gfn + i);
721 }
722
723 if (fatal_signal_pending(current))
724 return;
725 cond_resched();
726 }
727 }
728
729 /* Section: vm related */
730 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
731
732 /*
733 * Get (and clear) the dirty memory log for a memory slot.
734 */
kvm_vm_ioctl_get_dirty_log(struct kvm * kvm,struct kvm_dirty_log * log)735 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
736 struct kvm_dirty_log *log)
737 {
738 int r;
739 unsigned long n;
740 struct kvm_memory_slot *memslot;
741 int is_dirty;
742
743 if (kvm_is_ucontrol(kvm))
744 return -EINVAL;
745
746 mutex_lock(&kvm->slots_lock);
747
748 r = -EINVAL;
749 if (log->slot >= KVM_USER_MEM_SLOTS)
750 goto out;
751
752 r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
753 if (r)
754 goto out;
755
756 /* Clear the dirty log */
757 if (is_dirty) {
758 n = kvm_dirty_bitmap_bytes(memslot);
759 memset(memslot->dirty_bitmap, 0, n);
760 }
761 r = 0;
762 out:
763 mutex_unlock(&kvm->slots_lock);
764 return r;
765 }
766
icpt_operexc_on_all_vcpus(struct kvm * kvm)767 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
768 {
769 unsigned long i;
770 struct kvm_vcpu *vcpu;
771
772 kvm_for_each_vcpu(i, vcpu, kvm) {
773 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
774 }
775 }
776
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)777 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
778 {
779 int r;
780
781 if (cap->flags)
782 return -EINVAL;
783
784 switch (cap->cap) {
785 case KVM_CAP_S390_IRQCHIP:
786 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
787 kvm->arch.use_irqchip = 1;
788 r = 0;
789 break;
790 case KVM_CAP_S390_USER_SIGP:
791 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
792 kvm->arch.user_sigp = 1;
793 r = 0;
794 break;
795 case KVM_CAP_S390_VECTOR_REGISTERS:
796 mutex_lock(&kvm->lock);
797 if (kvm->created_vcpus) {
798 r = -EBUSY;
799 } else if (cpu_has_vx()) {
800 set_kvm_facility(kvm->arch.model.fac_mask, 129);
801 set_kvm_facility(kvm->arch.model.fac_list, 129);
802 if (test_facility(134)) {
803 set_kvm_facility(kvm->arch.model.fac_mask, 134);
804 set_kvm_facility(kvm->arch.model.fac_list, 134);
805 }
806 if (test_facility(135)) {
807 set_kvm_facility(kvm->arch.model.fac_mask, 135);
808 set_kvm_facility(kvm->arch.model.fac_list, 135);
809 }
810 if (test_facility(148)) {
811 set_kvm_facility(kvm->arch.model.fac_mask, 148);
812 set_kvm_facility(kvm->arch.model.fac_list, 148);
813 }
814 if (test_facility(152)) {
815 set_kvm_facility(kvm->arch.model.fac_mask, 152);
816 set_kvm_facility(kvm->arch.model.fac_list, 152);
817 }
818 if (test_facility(192)) {
819 set_kvm_facility(kvm->arch.model.fac_mask, 192);
820 set_kvm_facility(kvm->arch.model.fac_list, 192);
821 }
822 if (test_facility(198)) {
823 set_kvm_facility(kvm->arch.model.fac_mask, 198);
824 set_kvm_facility(kvm->arch.model.fac_list, 198);
825 }
826 if (test_facility(199)) {
827 set_kvm_facility(kvm->arch.model.fac_mask, 199);
828 set_kvm_facility(kvm->arch.model.fac_list, 199);
829 }
830 r = 0;
831 } else
832 r = -EINVAL;
833 mutex_unlock(&kvm->lock);
834 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
835 r ? "(not available)" : "(success)");
836 break;
837 case KVM_CAP_S390_RI:
838 r = -EINVAL;
839 mutex_lock(&kvm->lock);
840 if (kvm->created_vcpus) {
841 r = -EBUSY;
842 } else if (test_facility(64)) {
843 set_kvm_facility(kvm->arch.model.fac_mask, 64);
844 set_kvm_facility(kvm->arch.model.fac_list, 64);
845 r = 0;
846 }
847 mutex_unlock(&kvm->lock);
848 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
849 r ? "(not available)" : "(success)");
850 break;
851 case KVM_CAP_S390_AIS:
852 mutex_lock(&kvm->lock);
853 if (kvm->created_vcpus) {
854 r = -EBUSY;
855 } else {
856 set_kvm_facility(kvm->arch.model.fac_mask, 72);
857 set_kvm_facility(kvm->arch.model.fac_list, 72);
858 r = 0;
859 }
860 mutex_unlock(&kvm->lock);
861 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
862 r ? "(not available)" : "(success)");
863 break;
864 case KVM_CAP_S390_GS:
865 r = -EINVAL;
866 mutex_lock(&kvm->lock);
867 if (kvm->created_vcpus) {
868 r = -EBUSY;
869 } else if (test_facility(133)) {
870 set_kvm_facility(kvm->arch.model.fac_mask, 133);
871 set_kvm_facility(kvm->arch.model.fac_list, 133);
872 r = 0;
873 }
874 mutex_unlock(&kvm->lock);
875 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
876 r ? "(not available)" : "(success)");
877 break;
878 case KVM_CAP_S390_HPAGE_1M:
879 mutex_lock(&kvm->lock);
880 if (kvm->created_vcpus)
881 r = -EBUSY;
882 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
883 r = -EINVAL;
884 else {
885 r = 0;
886 mmap_write_lock(kvm->mm);
887 kvm->mm->context.allow_gmap_hpage_1m = 1;
888 mmap_write_unlock(kvm->mm);
889 /*
890 * We might have to create fake 4k page
891 * tables. To avoid that the hardware works on
892 * stale PGSTEs, we emulate these instructions.
893 */
894 kvm->arch.use_skf = 0;
895 kvm->arch.use_pfmfi = 0;
896 }
897 mutex_unlock(&kvm->lock);
898 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
899 r ? "(not available)" : "(success)");
900 break;
901 case KVM_CAP_S390_USER_STSI:
902 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
903 kvm->arch.user_stsi = 1;
904 r = 0;
905 break;
906 case KVM_CAP_S390_USER_INSTR0:
907 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
908 kvm->arch.user_instr0 = 1;
909 icpt_operexc_on_all_vcpus(kvm);
910 r = 0;
911 break;
912 case KVM_CAP_S390_CPU_TOPOLOGY:
913 r = -EINVAL;
914 mutex_lock(&kvm->lock);
915 if (kvm->created_vcpus) {
916 r = -EBUSY;
917 } else if (test_facility(11)) {
918 set_kvm_facility(kvm->arch.model.fac_mask, 11);
919 set_kvm_facility(kvm->arch.model.fac_list, 11);
920 r = 0;
921 }
922 mutex_unlock(&kvm->lock);
923 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",
924 r ? "(not available)" : "(success)");
925 break;
926 case KVM_CAP_S390_USER_OPEREXEC:
927 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_OPEREXEC");
928 kvm->arch.user_operexec = 1;
929 icpt_operexc_on_all_vcpus(kvm);
930 r = 0;
931 break;
932 default:
933 r = -EINVAL;
934 break;
935 }
936 return r;
937 }
938
kvm_s390_get_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)939 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
940 {
941 int ret;
942
943 switch (attr->attr) {
944 case KVM_S390_VM_MEM_LIMIT_SIZE:
945 ret = 0;
946 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
947 kvm->arch.mem_limit);
948 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
949 ret = -EFAULT;
950 break;
951 default:
952 ret = -ENXIO;
953 break;
954 }
955 return ret;
956 }
957
kvm_s390_set_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)958 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
959 {
960 int ret;
961 unsigned int idx;
962 switch (attr->attr) {
963 case KVM_S390_VM_MEM_ENABLE_CMMA:
964 ret = -ENXIO;
965 if (!sclp.has_cmma)
966 break;
967
968 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
969 mutex_lock(&kvm->lock);
970 if (kvm->created_vcpus)
971 ret = -EBUSY;
972 else if (kvm->mm->context.allow_gmap_hpage_1m)
973 ret = -EINVAL;
974 else {
975 kvm->arch.use_cmma = 1;
976 /* Not compatible with cmma. */
977 kvm->arch.use_pfmfi = 0;
978 ret = 0;
979 }
980 mutex_unlock(&kvm->lock);
981 break;
982 case KVM_S390_VM_MEM_CLR_CMMA:
983 ret = -ENXIO;
984 if (!sclp.has_cmma)
985 break;
986 ret = -EINVAL;
987 if (!kvm->arch.use_cmma)
988 break;
989
990 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
991 mutex_lock(&kvm->lock);
992 idx = srcu_read_lock(&kvm->srcu);
993 s390_reset_cmma(kvm->arch.gmap->mm);
994 srcu_read_unlock(&kvm->srcu, idx);
995 mutex_unlock(&kvm->lock);
996 ret = 0;
997 break;
998 case KVM_S390_VM_MEM_LIMIT_SIZE: {
999 unsigned long new_limit;
1000
1001 if (kvm_is_ucontrol(kvm))
1002 return -EINVAL;
1003
1004 if (get_user(new_limit, (u64 __user *)attr->addr))
1005 return -EFAULT;
1006
1007 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
1008 new_limit > kvm->arch.mem_limit)
1009 return -E2BIG;
1010
1011 if (!new_limit)
1012 return -EINVAL;
1013
1014 /* gmap_create takes last usable address */
1015 if (new_limit != KVM_S390_NO_MEM_LIMIT)
1016 new_limit -= 1;
1017
1018 ret = -EBUSY;
1019 mutex_lock(&kvm->lock);
1020 if (!kvm->created_vcpus) {
1021 /* gmap_create will round the limit up */
1022 struct gmap *new = gmap_create(current->mm, new_limit);
1023
1024 if (!new) {
1025 ret = -ENOMEM;
1026 } else {
1027 gmap_remove(kvm->arch.gmap);
1028 new->private = kvm;
1029 kvm->arch.gmap = new;
1030 ret = 0;
1031 }
1032 }
1033 mutex_unlock(&kvm->lock);
1034 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
1035 VM_EVENT(kvm, 3, "New guest asce: 0x%p",
1036 (void *) kvm->arch.gmap->asce);
1037 break;
1038 }
1039 default:
1040 ret = -ENXIO;
1041 break;
1042 }
1043 return ret;
1044 }
1045
1046 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
1047
kvm_s390_vcpu_crypto_reset_all(struct kvm * kvm)1048 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
1049 {
1050 struct kvm_vcpu *vcpu;
1051 unsigned long i;
1052
1053 kvm_s390_vcpu_block_all(kvm);
1054
1055 kvm_for_each_vcpu(i, vcpu, kvm) {
1056 kvm_s390_vcpu_crypto_setup(vcpu);
1057 /* recreate the shadow crycb by leaving the VSIE handler */
1058 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1059 }
1060
1061 kvm_s390_vcpu_unblock_all(kvm);
1062 }
1063
kvm_s390_vm_set_crypto(struct kvm * kvm,struct kvm_device_attr * attr)1064 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
1065 {
1066 mutex_lock(&kvm->lock);
1067 switch (attr->attr) {
1068 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1069 if (!test_kvm_facility(kvm, 76)) {
1070 mutex_unlock(&kvm->lock);
1071 return -EINVAL;
1072 }
1073 get_random_bytes(
1074 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1075 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1076 kvm->arch.crypto.aes_kw = 1;
1077 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
1078 break;
1079 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1080 if (!test_kvm_facility(kvm, 76)) {
1081 mutex_unlock(&kvm->lock);
1082 return -EINVAL;
1083 }
1084 get_random_bytes(
1085 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1086 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1087 kvm->arch.crypto.dea_kw = 1;
1088 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
1089 break;
1090 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1091 if (!test_kvm_facility(kvm, 76)) {
1092 mutex_unlock(&kvm->lock);
1093 return -EINVAL;
1094 }
1095 kvm->arch.crypto.aes_kw = 0;
1096 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
1097 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1098 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
1099 break;
1100 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1101 if (!test_kvm_facility(kvm, 76)) {
1102 mutex_unlock(&kvm->lock);
1103 return -EINVAL;
1104 }
1105 kvm->arch.crypto.dea_kw = 0;
1106 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
1107 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1108 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
1109 break;
1110 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1111 if (!ap_instructions_available()) {
1112 mutex_unlock(&kvm->lock);
1113 return -EOPNOTSUPP;
1114 }
1115 kvm->arch.crypto.apie = 1;
1116 break;
1117 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1118 if (!ap_instructions_available()) {
1119 mutex_unlock(&kvm->lock);
1120 return -EOPNOTSUPP;
1121 }
1122 kvm->arch.crypto.apie = 0;
1123 break;
1124 default:
1125 mutex_unlock(&kvm->lock);
1126 return -ENXIO;
1127 }
1128
1129 kvm_s390_vcpu_crypto_reset_all(kvm);
1130 mutex_unlock(&kvm->lock);
1131 return 0;
1132 }
1133
kvm_s390_vcpu_pci_setup(struct kvm_vcpu * vcpu)1134 static void kvm_s390_vcpu_pci_setup(struct kvm_vcpu *vcpu)
1135 {
1136 /* Only set the ECB bits after guest requests zPCI interpretation */
1137 if (!vcpu->kvm->arch.use_zpci_interp)
1138 return;
1139
1140 vcpu->arch.sie_block->ecb2 |= ECB2_ZPCI_LSI;
1141 vcpu->arch.sie_block->ecb3 |= ECB3_AISII + ECB3_AISI;
1142 }
1143
kvm_s390_vcpu_pci_enable_interp(struct kvm * kvm)1144 void kvm_s390_vcpu_pci_enable_interp(struct kvm *kvm)
1145 {
1146 struct kvm_vcpu *vcpu;
1147 unsigned long i;
1148
1149 lockdep_assert_held(&kvm->lock);
1150
1151 if (!kvm_s390_pci_interp_allowed())
1152 return;
1153
1154 /*
1155 * If host is configured for PCI and the necessary facilities are
1156 * available, turn on interpretation for the life of this guest
1157 */
1158 kvm->arch.use_zpci_interp = 1;
1159
1160 kvm_s390_vcpu_block_all(kvm);
1161
1162 kvm_for_each_vcpu(i, vcpu, kvm) {
1163 kvm_s390_vcpu_pci_setup(vcpu);
1164 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1165 }
1166
1167 kvm_s390_vcpu_unblock_all(kvm);
1168 }
1169
kvm_s390_sync_request_broadcast(struct kvm * kvm,int req)1170 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
1171 {
1172 unsigned long cx;
1173 struct kvm_vcpu *vcpu;
1174
1175 kvm_for_each_vcpu(cx, vcpu, kvm)
1176 kvm_s390_sync_request(req, vcpu);
1177 }
1178
1179 /*
1180 * Must be called with kvm->srcu held to avoid races on memslots, and with
1181 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
1182 */
kvm_s390_vm_start_migration(struct kvm * kvm)1183 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1184 {
1185 struct kvm_memory_slot *ms;
1186 struct kvm_memslots *slots;
1187 unsigned long ram_pages = 0;
1188 int bkt;
1189
1190 /* migration mode already enabled */
1191 if (kvm->arch.migration_mode)
1192 return 0;
1193 slots = kvm_memslots(kvm);
1194 if (!slots || kvm_memslots_empty(slots))
1195 return -EINVAL;
1196
1197 if (!kvm->arch.use_cmma) {
1198 kvm->arch.migration_mode = 1;
1199 return 0;
1200 }
1201 /* mark all the pages in active slots as dirty */
1202 kvm_for_each_memslot(ms, bkt, slots) {
1203 if (!ms->dirty_bitmap)
1204 return -EINVAL;
1205 /*
1206 * The second half of the bitmap is only used on x86,
1207 * and would be wasted otherwise, so we put it to good
1208 * use here to keep track of the state of the storage
1209 * attributes.
1210 */
1211 memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
1212 ram_pages += ms->npages;
1213 }
1214 atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
1215 kvm->arch.migration_mode = 1;
1216 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1217 return 0;
1218 }
1219
1220 /*
1221 * Must be called with kvm->slots_lock to avoid races with ourselves and
1222 * kvm_s390_vm_start_migration.
1223 */
kvm_s390_vm_stop_migration(struct kvm * kvm)1224 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1225 {
1226 /* migration mode already disabled */
1227 if (!kvm->arch.migration_mode)
1228 return 0;
1229 kvm->arch.migration_mode = 0;
1230 if (kvm->arch.use_cmma)
1231 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1232 return 0;
1233 }
1234
kvm_s390_vm_set_migration(struct kvm * kvm,struct kvm_device_attr * attr)1235 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1236 struct kvm_device_attr *attr)
1237 {
1238 int res = -ENXIO;
1239
1240 mutex_lock(&kvm->slots_lock);
1241 switch (attr->attr) {
1242 case KVM_S390_VM_MIGRATION_START:
1243 res = kvm_s390_vm_start_migration(kvm);
1244 break;
1245 case KVM_S390_VM_MIGRATION_STOP:
1246 res = kvm_s390_vm_stop_migration(kvm);
1247 break;
1248 default:
1249 break;
1250 }
1251 mutex_unlock(&kvm->slots_lock);
1252
1253 return res;
1254 }
1255
kvm_s390_vm_get_migration(struct kvm * kvm,struct kvm_device_attr * attr)1256 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1257 struct kvm_device_attr *attr)
1258 {
1259 u64 mig = kvm->arch.migration_mode;
1260
1261 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1262 return -ENXIO;
1263
1264 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1265 return -EFAULT;
1266 return 0;
1267 }
1268
1269 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod);
1270
kvm_s390_set_tod_ext(struct kvm * kvm,struct kvm_device_attr * attr)1271 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1272 {
1273 struct kvm_s390_vm_tod_clock gtod;
1274
1275 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
1276 return -EFAULT;
1277
1278 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1279 return -EINVAL;
1280 __kvm_s390_set_tod_clock(kvm, >od);
1281
1282 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1283 gtod.epoch_idx, gtod.tod);
1284
1285 return 0;
1286 }
1287
kvm_s390_set_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)1288 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1289 {
1290 u8 gtod_high;
1291
1292 if (copy_from_user(>od_high, (void __user *)attr->addr,
1293 sizeof(gtod_high)))
1294 return -EFAULT;
1295
1296 if (gtod_high != 0)
1297 return -EINVAL;
1298 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1299
1300 return 0;
1301 }
1302
kvm_s390_set_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)1303 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1304 {
1305 struct kvm_s390_vm_tod_clock gtod = { 0 };
1306
1307 if (copy_from_user(>od.tod, (void __user *)attr->addr,
1308 sizeof(gtod.tod)))
1309 return -EFAULT;
1310
1311 __kvm_s390_set_tod_clock(kvm, >od);
1312 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1313 return 0;
1314 }
1315
kvm_s390_set_tod(struct kvm * kvm,struct kvm_device_attr * attr)1316 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1317 {
1318 int ret;
1319
1320 if (attr->flags)
1321 return -EINVAL;
1322
1323 mutex_lock(&kvm->lock);
1324 /*
1325 * For protected guests, the TOD is managed by the ultravisor, so trying
1326 * to change it will never bring the expected results.
1327 */
1328 if (kvm_s390_pv_is_protected(kvm)) {
1329 ret = -EOPNOTSUPP;
1330 goto out_unlock;
1331 }
1332
1333 switch (attr->attr) {
1334 case KVM_S390_VM_TOD_EXT:
1335 ret = kvm_s390_set_tod_ext(kvm, attr);
1336 break;
1337 case KVM_S390_VM_TOD_HIGH:
1338 ret = kvm_s390_set_tod_high(kvm, attr);
1339 break;
1340 case KVM_S390_VM_TOD_LOW:
1341 ret = kvm_s390_set_tod_low(kvm, attr);
1342 break;
1343 default:
1344 ret = -ENXIO;
1345 break;
1346 }
1347
1348 out_unlock:
1349 mutex_unlock(&kvm->lock);
1350 return ret;
1351 }
1352
kvm_s390_get_tod_clock(struct kvm * kvm,struct kvm_s390_vm_tod_clock * gtod)1353 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1354 struct kvm_s390_vm_tod_clock *gtod)
1355 {
1356 union tod_clock clk;
1357
1358 preempt_disable();
1359
1360 store_tod_clock_ext(&clk);
1361
1362 gtod->tod = clk.tod + kvm->arch.epoch;
1363 gtod->epoch_idx = 0;
1364 if (test_kvm_facility(kvm, 139)) {
1365 gtod->epoch_idx = clk.ei + kvm->arch.epdx;
1366 if (gtod->tod < clk.tod)
1367 gtod->epoch_idx += 1;
1368 }
1369
1370 preempt_enable();
1371 }
1372
kvm_s390_get_tod_ext(struct kvm * kvm,struct kvm_device_attr * attr)1373 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1374 {
1375 struct kvm_s390_vm_tod_clock gtod;
1376
1377 memset(>od, 0, sizeof(gtod));
1378 kvm_s390_get_tod_clock(kvm, >od);
1379 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1380 return -EFAULT;
1381
1382 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1383 gtod.epoch_idx, gtod.tod);
1384 return 0;
1385 }
1386
kvm_s390_get_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)1387 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1388 {
1389 u8 gtod_high = 0;
1390
1391 if (copy_to_user((void __user *)attr->addr, >od_high,
1392 sizeof(gtod_high)))
1393 return -EFAULT;
1394 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1395
1396 return 0;
1397 }
1398
kvm_s390_get_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)1399 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1400 {
1401 u64 gtod;
1402
1403 gtod = kvm_s390_get_tod_clock_fast(kvm);
1404 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1405 return -EFAULT;
1406 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1407
1408 return 0;
1409 }
1410
kvm_s390_get_tod(struct kvm * kvm,struct kvm_device_attr * attr)1411 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1412 {
1413 int ret;
1414
1415 if (attr->flags)
1416 return -EINVAL;
1417
1418 switch (attr->attr) {
1419 case KVM_S390_VM_TOD_EXT:
1420 ret = kvm_s390_get_tod_ext(kvm, attr);
1421 break;
1422 case KVM_S390_VM_TOD_HIGH:
1423 ret = kvm_s390_get_tod_high(kvm, attr);
1424 break;
1425 case KVM_S390_VM_TOD_LOW:
1426 ret = kvm_s390_get_tod_low(kvm, attr);
1427 break;
1428 default:
1429 ret = -ENXIO;
1430 break;
1431 }
1432 return ret;
1433 }
1434
kvm_s390_set_processor(struct kvm * kvm,struct kvm_device_attr * attr)1435 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1436 {
1437 struct kvm_s390_vm_cpu_processor *proc;
1438 u16 lowest_ibc, unblocked_ibc;
1439 int ret = 0;
1440
1441 mutex_lock(&kvm->lock);
1442 if (kvm->created_vcpus) {
1443 ret = -EBUSY;
1444 goto out;
1445 }
1446 proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
1447 if (!proc) {
1448 ret = -ENOMEM;
1449 goto out;
1450 }
1451 if (!copy_from_user(proc, (void __user *)attr->addr,
1452 sizeof(*proc))) {
1453 kvm->arch.model.cpuid = proc->cpuid;
1454 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1455 unblocked_ibc = sclp.ibc & 0xfff;
1456 if (lowest_ibc && proc->ibc) {
1457 if (proc->ibc > unblocked_ibc)
1458 kvm->arch.model.ibc = unblocked_ibc;
1459 else if (proc->ibc < lowest_ibc)
1460 kvm->arch.model.ibc = lowest_ibc;
1461 else
1462 kvm->arch.model.ibc = proc->ibc;
1463 }
1464 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1465 S390_ARCH_FAC_LIST_SIZE_BYTE);
1466 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1467 kvm->arch.model.ibc,
1468 kvm->arch.model.cpuid);
1469 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1470 kvm->arch.model.fac_list[0],
1471 kvm->arch.model.fac_list[1],
1472 kvm->arch.model.fac_list[2]);
1473 } else
1474 ret = -EFAULT;
1475 kfree(proc);
1476 out:
1477 mutex_unlock(&kvm->lock);
1478 return ret;
1479 }
1480
kvm_s390_set_processor_feat(struct kvm * kvm,struct kvm_device_attr * attr)1481 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1482 struct kvm_device_attr *attr)
1483 {
1484 struct kvm_s390_vm_cpu_feat data;
1485
1486 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1487 return -EFAULT;
1488 if (!bitmap_subset((unsigned long *) data.feat,
1489 kvm_s390_available_cpu_feat,
1490 KVM_S390_VM_CPU_FEAT_NR_BITS))
1491 return -EINVAL;
1492
1493 mutex_lock(&kvm->lock);
1494 if (kvm->created_vcpus) {
1495 mutex_unlock(&kvm->lock);
1496 return -EBUSY;
1497 }
1498 bitmap_from_arr64(kvm->arch.cpu_feat, data.feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1499 mutex_unlock(&kvm->lock);
1500 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1501 data.feat[0],
1502 data.feat[1],
1503 data.feat[2]);
1504 return 0;
1505 }
1506
kvm_s390_set_processor_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1507 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1508 struct kvm_device_attr *attr)
1509 {
1510 mutex_lock(&kvm->lock);
1511 if (kvm->created_vcpus) {
1512 mutex_unlock(&kvm->lock);
1513 return -EBUSY;
1514 }
1515
1516 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1517 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1518 mutex_unlock(&kvm->lock);
1519 return -EFAULT;
1520 }
1521 mutex_unlock(&kvm->lock);
1522
1523 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1524 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1525 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1526 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1527 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1528 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1529 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1530 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1531 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1532 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1533 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1534 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1535 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1536 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1537 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1538 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1539 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1540 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1541 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1542 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1543 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1544 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1545 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1546 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1547 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1548 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1549 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1550 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1551 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1552 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1553 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1554 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1555 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1556 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1557 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1558 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1559 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1560 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1561 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1562 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1563 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1564 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1565 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1566 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1567 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1568 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1569 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1570 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1571 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1572 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1573 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1574 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1575 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1576 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1577 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1578 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1579 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1580 VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1581 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1582 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1583
1584 return 0;
1585 }
1586
1587 #define KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK \
1588 ( \
1589 ((struct kvm_s390_vm_cpu_uv_feat){ \
1590 .ap = 1, \
1591 .ap_intr = 1, \
1592 }) \
1593 .feat \
1594 )
1595
kvm_s390_set_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1596 static int kvm_s390_set_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1597 {
1598 struct kvm_s390_vm_cpu_uv_feat __user *ptr = (void __user *)attr->addr;
1599 unsigned long data, filter;
1600
1601 filter = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1602 if (get_user(data, &ptr->feat))
1603 return -EFAULT;
1604 if (!bitmap_subset(&data, &filter, KVM_S390_VM_CPU_UV_FEAT_NR_BITS))
1605 return -EINVAL;
1606
1607 mutex_lock(&kvm->lock);
1608 if (kvm->created_vcpus) {
1609 mutex_unlock(&kvm->lock);
1610 return -EBUSY;
1611 }
1612 kvm->arch.model.uv_feat_guest.feat = data;
1613 mutex_unlock(&kvm->lock);
1614
1615 VM_EVENT(kvm, 3, "SET: guest UV-feat: 0x%16.16lx", data);
1616
1617 return 0;
1618 }
1619
kvm_s390_set_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)1620 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1621 {
1622 int ret = -ENXIO;
1623
1624 switch (attr->attr) {
1625 case KVM_S390_VM_CPU_PROCESSOR:
1626 ret = kvm_s390_set_processor(kvm, attr);
1627 break;
1628 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1629 ret = kvm_s390_set_processor_feat(kvm, attr);
1630 break;
1631 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1632 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1633 break;
1634 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1635 ret = kvm_s390_set_uv_feat(kvm, attr);
1636 break;
1637 }
1638 return ret;
1639 }
1640
kvm_s390_get_processor(struct kvm * kvm,struct kvm_device_attr * attr)1641 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1642 {
1643 struct kvm_s390_vm_cpu_processor *proc;
1644 int ret = 0;
1645
1646 proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
1647 if (!proc) {
1648 ret = -ENOMEM;
1649 goto out;
1650 }
1651 proc->cpuid = kvm->arch.model.cpuid;
1652 proc->ibc = kvm->arch.model.ibc;
1653 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1654 S390_ARCH_FAC_LIST_SIZE_BYTE);
1655 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1656 kvm->arch.model.ibc,
1657 kvm->arch.model.cpuid);
1658 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1659 kvm->arch.model.fac_list[0],
1660 kvm->arch.model.fac_list[1],
1661 kvm->arch.model.fac_list[2]);
1662 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1663 ret = -EFAULT;
1664 kfree(proc);
1665 out:
1666 return ret;
1667 }
1668
kvm_s390_get_machine(struct kvm * kvm,struct kvm_device_attr * attr)1669 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1670 {
1671 struct kvm_s390_vm_cpu_machine *mach;
1672 int ret = 0;
1673
1674 mach = kzalloc(sizeof(*mach), GFP_KERNEL_ACCOUNT);
1675 if (!mach) {
1676 ret = -ENOMEM;
1677 goto out;
1678 }
1679 get_cpu_id((struct cpuid *) &mach->cpuid);
1680 mach->ibc = sclp.ibc;
1681 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1682 S390_ARCH_FAC_LIST_SIZE_BYTE);
1683 memcpy((unsigned long *)&mach->fac_list, stfle_fac_list,
1684 sizeof(stfle_fac_list));
1685 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1686 kvm->arch.model.ibc,
1687 kvm->arch.model.cpuid);
1688 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1689 mach->fac_mask[0],
1690 mach->fac_mask[1],
1691 mach->fac_mask[2]);
1692 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1693 mach->fac_list[0],
1694 mach->fac_list[1],
1695 mach->fac_list[2]);
1696 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1697 ret = -EFAULT;
1698 kfree(mach);
1699 out:
1700 return ret;
1701 }
1702
kvm_s390_get_processor_feat(struct kvm * kvm,struct kvm_device_attr * attr)1703 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1704 struct kvm_device_attr *attr)
1705 {
1706 struct kvm_s390_vm_cpu_feat data;
1707
1708 bitmap_to_arr64(data.feat, kvm->arch.cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1709 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1710 return -EFAULT;
1711 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1712 data.feat[0],
1713 data.feat[1],
1714 data.feat[2]);
1715 return 0;
1716 }
1717
kvm_s390_get_machine_feat(struct kvm * kvm,struct kvm_device_attr * attr)1718 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1719 struct kvm_device_attr *attr)
1720 {
1721 struct kvm_s390_vm_cpu_feat data;
1722
1723 bitmap_to_arr64(data.feat, kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1724 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1725 return -EFAULT;
1726 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1727 data.feat[0],
1728 data.feat[1],
1729 data.feat[2]);
1730 return 0;
1731 }
1732
kvm_s390_get_processor_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1733 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1734 struct kvm_device_attr *attr)
1735 {
1736 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1737 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1738 return -EFAULT;
1739
1740 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1741 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1742 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1743 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1744 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1745 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1746 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1747 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1748 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1749 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1750 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1751 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1752 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1753 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1754 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1755 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1756 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1757 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1758 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1759 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1760 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1761 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1762 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1763 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1764 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1765 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1766 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1767 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1768 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1769 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1770 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1771 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1772 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1773 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1774 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1775 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1776 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1777 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1778 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1779 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1780 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1781 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1782 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1783 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1784 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1785 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1786 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1787 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1788 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1789 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1790 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1791 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1792 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1793 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1794 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1795 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1796 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1797 VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1798 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1799 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1800
1801 return 0;
1802 }
1803
kvm_s390_get_machine_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1804 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1805 struct kvm_device_attr *attr)
1806 {
1807 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1808 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1809 return -EFAULT;
1810
1811 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1812 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1813 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1814 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1815 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1816 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1817 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1818 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1819 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1820 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1821 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1822 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1823 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1824 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1825 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1826 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1827 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1828 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1829 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1830 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1831 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1832 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1833 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1834 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1835 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1836 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1837 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1838 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1839 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1840 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1841 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1842 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1843 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1844 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1845 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1846 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1847 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1848 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1849 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1850 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1851 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1852 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1853 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1854 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1855 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1856 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1857 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1858 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1859 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1860 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1861 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1862 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1863 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1864 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1865 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1866 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1867 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1868 VM_EVENT(kvm, 3, "GET: host PFCR subfunc 0x%16.16lx.%16.16lx",
1869 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1870 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1871
1872 return 0;
1873 }
1874
kvm_s390_get_processor_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1875 static int kvm_s390_get_processor_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1876 {
1877 struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1878 unsigned long feat = kvm->arch.model.uv_feat_guest.feat;
1879
1880 if (put_user(feat, &dst->feat))
1881 return -EFAULT;
1882 VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1883
1884 return 0;
1885 }
1886
kvm_s390_get_machine_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1887 static int kvm_s390_get_machine_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1888 {
1889 struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1890 unsigned long feat;
1891
1892 BUILD_BUG_ON(sizeof(*dst) != sizeof(uv_info.uv_feature_indications));
1893
1894 feat = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1895 if (put_user(feat, &dst->feat))
1896 return -EFAULT;
1897 VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1898
1899 return 0;
1900 }
1901
kvm_s390_get_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)1902 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1903 {
1904 int ret = -ENXIO;
1905
1906 switch (attr->attr) {
1907 case KVM_S390_VM_CPU_PROCESSOR:
1908 ret = kvm_s390_get_processor(kvm, attr);
1909 break;
1910 case KVM_S390_VM_CPU_MACHINE:
1911 ret = kvm_s390_get_machine(kvm, attr);
1912 break;
1913 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1914 ret = kvm_s390_get_processor_feat(kvm, attr);
1915 break;
1916 case KVM_S390_VM_CPU_MACHINE_FEAT:
1917 ret = kvm_s390_get_machine_feat(kvm, attr);
1918 break;
1919 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1920 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1921 break;
1922 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1923 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1924 break;
1925 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1926 ret = kvm_s390_get_processor_uv_feat(kvm, attr);
1927 break;
1928 case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
1929 ret = kvm_s390_get_machine_uv_feat(kvm, attr);
1930 break;
1931 }
1932 return ret;
1933 }
1934
1935 /**
1936 * kvm_s390_update_topology_change_report - update CPU topology change report
1937 * @kvm: guest KVM description
1938 * @val: set or clear the MTCR bit
1939 *
1940 * Updates the Multiprocessor Topology-Change-Report bit to signal
1941 * the guest with a topology change.
1942 * This is only relevant if the topology facility is present.
1943 */
kvm_s390_update_topology_change_report(struct kvm * kvm,bool val)1944 static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
1945 {
1946 union sca_utility new, old;
1947 struct esca_block *sca;
1948
1949 sca = kvm->arch.sca;
1950 old = READ_ONCE(sca->utility);
1951 do {
1952 new = old;
1953 new.mtcr = val;
1954 } while (!try_cmpxchg(&sca->utility.val, &old.val, new.val));
1955 }
1956
kvm_s390_set_topo_change_indication(struct kvm * kvm,struct kvm_device_attr * attr)1957 static int kvm_s390_set_topo_change_indication(struct kvm *kvm,
1958 struct kvm_device_attr *attr)
1959 {
1960 if (!test_kvm_facility(kvm, 11))
1961 return -ENXIO;
1962
1963 kvm_s390_update_topology_change_report(kvm, !!attr->attr);
1964 return 0;
1965 }
1966
kvm_s390_get_topo_change_indication(struct kvm * kvm,struct kvm_device_attr * attr)1967 static int kvm_s390_get_topo_change_indication(struct kvm *kvm,
1968 struct kvm_device_attr *attr)
1969 {
1970 u8 topo;
1971
1972 if (!test_kvm_facility(kvm, 11))
1973 return -ENXIO;
1974
1975 topo = kvm->arch.sca->utility.mtcr;
1976
1977 return put_user(topo, (u8 __user *)attr->addr);
1978 }
1979
kvm_s390_vm_set_attr(struct kvm * kvm,struct kvm_device_attr * attr)1980 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
1981 {
1982 int ret;
1983
1984 switch (attr->group) {
1985 case KVM_S390_VM_MEM_CTRL:
1986 ret = kvm_s390_set_mem_control(kvm, attr);
1987 break;
1988 case KVM_S390_VM_TOD:
1989 ret = kvm_s390_set_tod(kvm, attr);
1990 break;
1991 case KVM_S390_VM_CPU_MODEL:
1992 ret = kvm_s390_set_cpu_model(kvm, attr);
1993 break;
1994 case KVM_S390_VM_CRYPTO:
1995 ret = kvm_s390_vm_set_crypto(kvm, attr);
1996 break;
1997 case KVM_S390_VM_MIGRATION:
1998 ret = kvm_s390_vm_set_migration(kvm, attr);
1999 break;
2000 case KVM_S390_VM_CPU_TOPOLOGY:
2001 ret = kvm_s390_set_topo_change_indication(kvm, attr);
2002 break;
2003 default:
2004 ret = -ENXIO;
2005 break;
2006 }
2007
2008 return ret;
2009 }
2010
kvm_s390_vm_get_attr(struct kvm * kvm,struct kvm_device_attr * attr)2011 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2012 {
2013 int ret;
2014
2015 switch (attr->group) {
2016 case KVM_S390_VM_MEM_CTRL:
2017 ret = kvm_s390_get_mem_control(kvm, attr);
2018 break;
2019 case KVM_S390_VM_TOD:
2020 ret = kvm_s390_get_tod(kvm, attr);
2021 break;
2022 case KVM_S390_VM_CPU_MODEL:
2023 ret = kvm_s390_get_cpu_model(kvm, attr);
2024 break;
2025 case KVM_S390_VM_MIGRATION:
2026 ret = kvm_s390_vm_get_migration(kvm, attr);
2027 break;
2028 case KVM_S390_VM_CPU_TOPOLOGY:
2029 ret = kvm_s390_get_topo_change_indication(kvm, attr);
2030 break;
2031 default:
2032 ret = -ENXIO;
2033 break;
2034 }
2035
2036 return ret;
2037 }
2038
kvm_s390_vm_has_attr(struct kvm * kvm,struct kvm_device_attr * attr)2039 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2040 {
2041 int ret;
2042
2043 switch (attr->group) {
2044 case KVM_S390_VM_MEM_CTRL:
2045 switch (attr->attr) {
2046 case KVM_S390_VM_MEM_ENABLE_CMMA:
2047 case KVM_S390_VM_MEM_CLR_CMMA:
2048 ret = sclp.has_cmma ? 0 : -ENXIO;
2049 break;
2050 case KVM_S390_VM_MEM_LIMIT_SIZE:
2051 ret = 0;
2052 break;
2053 default:
2054 ret = -ENXIO;
2055 break;
2056 }
2057 break;
2058 case KVM_S390_VM_TOD:
2059 switch (attr->attr) {
2060 case KVM_S390_VM_TOD_LOW:
2061 case KVM_S390_VM_TOD_HIGH:
2062 ret = 0;
2063 break;
2064 default:
2065 ret = -ENXIO;
2066 break;
2067 }
2068 break;
2069 case KVM_S390_VM_CPU_MODEL:
2070 switch (attr->attr) {
2071 case KVM_S390_VM_CPU_PROCESSOR:
2072 case KVM_S390_VM_CPU_MACHINE:
2073 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
2074 case KVM_S390_VM_CPU_MACHINE_FEAT:
2075 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
2076 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
2077 case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
2078 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
2079 ret = 0;
2080 break;
2081 default:
2082 ret = -ENXIO;
2083 break;
2084 }
2085 break;
2086 case KVM_S390_VM_CRYPTO:
2087 switch (attr->attr) {
2088 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
2089 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
2090 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
2091 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
2092 ret = 0;
2093 break;
2094 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
2095 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
2096 ret = ap_instructions_available() ? 0 : -ENXIO;
2097 break;
2098 default:
2099 ret = -ENXIO;
2100 break;
2101 }
2102 break;
2103 case KVM_S390_VM_MIGRATION:
2104 ret = 0;
2105 break;
2106 case KVM_S390_VM_CPU_TOPOLOGY:
2107 ret = test_kvm_facility(kvm, 11) ? 0 : -ENXIO;
2108 break;
2109 default:
2110 ret = -ENXIO;
2111 break;
2112 }
2113
2114 return ret;
2115 }
2116
kvm_s390_get_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)2117 static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2118 {
2119 uint8_t *keys;
2120 uint64_t hva;
2121 int srcu_idx, i, r = 0;
2122
2123 if (args->flags != 0)
2124 return -EINVAL;
2125
2126 /* Is this guest using storage keys? */
2127 if (!mm_uses_skeys(current->mm))
2128 return KVM_S390_GET_SKEYS_NONE;
2129
2130 /* Enforce sane limit on memory allocation */
2131 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2132 return -EINVAL;
2133
2134 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
2135 if (!keys)
2136 return -ENOMEM;
2137
2138 mmap_read_lock(current->mm);
2139 srcu_idx = srcu_read_lock(&kvm->srcu);
2140 for (i = 0; i < args->count; i++) {
2141 hva = gfn_to_hva(kvm, args->start_gfn + i);
2142 if (kvm_is_error_hva(hva)) {
2143 r = -EFAULT;
2144 break;
2145 }
2146
2147 r = get_guest_storage_key(current->mm, hva, &keys[i]);
2148 if (r)
2149 break;
2150 }
2151 srcu_read_unlock(&kvm->srcu, srcu_idx);
2152 mmap_read_unlock(current->mm);
2153
2154 if (!r) {
2155 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
2156 sizeof(uint8_t) * args->count);
2157 if (r)
2158 r = -EFAULT;
2159 }
2160
2161 kvfree(keys);
2162 return r;
2163 }
2164
kvm_s390_set_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)2165 static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2166 {
2167 uint8_t *keys;
2168 uint64_t hva;
2169 int srcu_idx, i, r = 0;
2170 bool unlocked;
2171
2172 if (args->flags != 0)
2173 return -EINVAL;
2174
2175 /* Enforce sane limit on memory allocation */
2176 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2177 return -EINVAL;
2178
2179 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
2180 if (!keys)
2181 return -ENOMEM;
2182
2183 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
2184 sizeof(uint8_t) * args->count);
2185 if (r) {
2186 r = -EFAULT;
2187 goto out;
2188 }
2189
2190 /* Enable storage key handling for the guest */
2191 r = s390_enable_skey();
2192 if (r)
2193 goto out;
2194
2195 i = 0;
2196 mmap_read_lock(current->mm);
2197 srcu_idx = srcu_read_lock(&kvm->srcu);
2198 while (i < args->count) {
2199 unlocked = false;
2200 hva = gfn_to_hva(kvm, args->start_gfn + i);
2201 if (kvm_is_error_hva(hva)) {
2202 r = -EFAULT;
2203 break;
2204 }
2205
2206 /* Lowest order bit is reserved */
2207 if (keys[i] & 0x01) {
2208 r = -EINVAL;
2209 break;
2210 }
2211
2212 r = set_guest_storage_key(current->mm, hva, keys[i], 0);
2213 if (r) {
2214 r = fixup_user_fault(current->mm, hva,
2215 FAULT_FLAG_WRITE, &unlocked);
2216 if (r)
2217 break;
2218 }
2219 if (!r)
2220 i++;
2221 }
2222 srcu_read_unlock(&kvm->srcu, srcu_idx);
2223 mmap_read_unlock(current->mm);
2224 out:
2225 kvfree(keys);
2226 return r;
2227 }
2228
2229 /*
2230 * Base address and length must be sent at the start of each block, therefore
2231 * it's cheaper to send some clean data, as long as it's less than the size of
2232 * two longs.
2233 */
2234 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
2235 /* for consistency */
2236 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
2237
kvm_s390_peek_cmma(struct kvm * kvm,struct kvm_s390_cmma_log * args,u8 * res,unsigned long bufsize)2238 static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2239 u8 *res, unsigned long bufsize)
2240 {
2241 unsigned long pgstev, hva, cur_gfn = args->start_gfn;
2242
2243 args->count = 0;
2244 while (args->count < bufsize) {
2245 hva = gfn_to_hva(kvm, cur_gfn);
2246 /*
2247 * We return an error if the first value was invalid, but we
2248 * return successfully if at least one value was copied.
2249 */
2250 if (kvm_is_error_hva(hva))
2251 return args->count ? 0 : -EFAULT;
2252 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2253 pgstev = 0;
2254 res[args->count++] = (pgstev >> 24) & 0x43;
2255 cur_gfn++;
2256 }
2257
2258 return 0;
2259 }
2260
gfn_to_memslot_approx(struct kvm_memslots * slots,gfn_t gfn)2261 static struct kvm_memory_slot *gfn_to_memslot_approx(struct kvm_memslots *slots,
2262 gfn_t gfn)
2263 {
2264 return ____gfn_to_memslot(slots, gfn, true);
2265 }
2266
kvm_s390_next_dirty_cmma(struct kvm_memslots * slots,unsigned long cur_gfn)2267 static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
2268 unsigned long cur_gfn)
2269 {
2270 struct kvm_memory_slot *ms = gfn_to_memslot_approx(slots, cur_gfn);
2271 unsigned long ofs = cur_gfn - ms->base_gfn;
2272 struct rb_node *mnode = &ms->gfn_node[slots->node_idx];
2273
2274 if (ms->base_gfn + ms->npages <= cur_gfn) {
2275 mnode = rb_next(mnode);
2276 /* If we are above the highest slot, wrap around */
2277 if (!mnode)
2278 mnode = rb_first(&slots->gfn_tree);
2279
2280 ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]);
2281 ofs = 0;
2282 }
2283
2284 if (cur_gfn < ms->base_gfn)
2285 ofs = 0;
2286
2287 ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
2288 while (ofs >= ms->npages && (mnode = rb_next(mnode))) {
2289 ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]);
2290 ofs = find_first_bit(kvm_second_dirty_bitmap(ms), ms->npages);
2291 }
2292 return ms->base_gfn + ofs;
2293 }
2294
kvm_s390_get_cmma(struct kvm * kvm,struct kvm_s390_cmma_log * args,u8 * res,unsigned long bufsize)2295 static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
2296 u8 *res, unsigned long bufsize)
2297 {
2298 unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
2299 struct kvm_memslots *slots = kvm_memslots(kvm);
2300 struct kvm_memory_slot *ms;
2301
2302 if (unlikely(kvm_memslots_empty(slots)))
2303 return 0;
2304
2305 cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
2306 ms = gfn_to_memslot(kvm, cur_gfn);
2307 args->count = 0;
2308 args->start_gfn = cur_gfn;
2309 if (!ms)
2310 return 0;
2311 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2312 mem_end = kvm_s390_get_gfn_end(slots);
2313
2314 while (args->count < bufsize) {
2315 hva = gfn_to_hva(kvm, cur_gfn);
2316 if (kvm_is_error_hva(hva))
2317 return 0;
2318 /* Decrement only if we actually flipped the bit to 0 */
2319 if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
2320 atomic64_dec(&kvm->arch.cmma_dirty_pages);
2321 if (get_pgste(kvm->mm, hva, &pgstev) < 0)
2322 pgstev = 0;
2323 /* Save the value */
2324 res[args->count++] = (pgstev >> 24) & 0x43;
2325 /* If the next bit is too far away, stop. */
2326 if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
2327 return 0;
2328 /* If we reached the previous "next", find the next one */
2329 if (cur_gfn == next_gfn)
2330 next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
2331 /* Reached the end of memory or of the buffer, stop */
2332 if ((next_gfn >= mem_end) ||
2333 (next_gfn - args->start_gfn >= bufsize))
2334 return 0;
2335 cur_gfn++;
2336 /* Reached the end of the current memslot, take the next one. */
2337 if (cur_gfn - ms->base_gfn >= ms->npages) {
2338 ms = gfn_to_memslot(kvm, cur_gfn);
2339 if (!ms)
2340 return 0;
2341 }
2342 }
2343 return 0;
2344 }
2345
2346 /*
2347 * This function searches for the next page with dirty CMMA attributes, and
2348 * saves the attributes in the buffer up to either the end of the buffer or
2349 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2350 * no trailing clean bytes are saved.
2351 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2352 * output buffer will indicate 0 as length.
2353 */
kvm_s390_get_cmma_bits(struct kvm * kvm,struct kvm_s390_cmma_log * args)2354 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2355 struct kvm_s390_cmma_log *args)
2356 {
2357 unsigned long bufsize;
2358 int srcu_idx, peek, ret;
2359 u8 *values;
2360
2361 if (!kvm->arch.use_cmma)
2362 return -ENXIO;
2363 /* Invalid/unsupported flags were specified */
2364 if (args->flags & ~KVM_S390_CMMA_PEEK)
2365 return -EINVAL;
2366 /* Migration mode query, and we are not doing a migration */
2367 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2368 if (!peek && !kvm->arch.migration_mode)
2369 return -EINVAL;
2370 /* CMMA is disabled or was not used, or the buffer has length zero */
2371 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2372 if (!bufsize || !kvm->mm->context.uses_cmm) {
2373 memset(args, 0, sizeof(*args));
2374 return 0;
2375 }
2376 /* We are not peeking, and there are no dirty pages */
2377 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2378 memset(args, 0, sizeof(*args));
2379 return 0;
2380 }
2381
2382 values = vmalloc(bufsize);
2383 if (!values)
2384 return -ENOMEM;
2385
2386 mmap_read_lock(kvm->mm);
2387 srcu_idx = srcu_read_lock(&kvm->srcu);
2388 if (peek)
2389 ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
2390 else
2391 ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
2392 srcu_read_unlock(&kvm->srcu, srcu_idx);
2393 mmap_read_unlock(kvm->mm);
2394
2395 if (kvm->arch.migration_mode)
2396 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2397 else
2398 args->remaining = 0;
2399
2400 if (copy_to_user((void __user *)args->values, values, args->count))
2401 ret = -EFAULT;
2402
2403 vfree(values);
2404 return ret;
2405 }
2406
2407 /*
2408 * This function sets the CMMA attributes for the given pages. If the input
2409 * buffer has zero length, no action is taken, otherwise the attributes are
2410 * set and the mm->context.uses_cmm flag is set.
2411 */
kvm_s390_set_cmma_bits(struct kvm * kvm,const struct kvm_s390_cmma_log * args)2412 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2413 const struct kvm_s390_cmma_log *args)
2414 {
2415 unsigned long hva, mask, pgstev, i;
2416 uint8_t *bits;
2417 int srcu_idx, r = 0;
2418
2419 mask = args->mask;
2420
2421 if (!kvm->arch.use_cmma)
2422 return -ENXIO;
2423 /* invalid/unsupported flags */
2424 if (args->flags != 0)
2425 return -EINVAL;
2426 /* Enforce sane limit on memory allocation */
2427 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2428 return -EINVAL;
2429 /* Nothing to do */
2430 if (args->count == 0)
2431 return 0;
2432
2433 bits = vmalloc(array_size(sizeof(*bits), args->count));
2434 if (!bits)
2435 return -ENOMEM;
2436
2437 r = copy_from_user(bits, (void __user *)args->values, args->count);
2438 if (r) {
2439 r = -EFAULT;
2440 goto out;
2441 }
2442
2443 mmap_read_lock(kvm->mm);
2444 srcu_idx = srcu_read_lock(&kvm->srcu);
2445 for (i = 0; i < args->count; i++) {
2446 hva = gfn_to_hva(kvm, args->start_gfn + i);
2447 if (kvm_is_error_hva(hva)) {
2448 r = -EFAULT;
2449 break;
2450 }
2451
2452 pgstev = bits[i];
2453 pgstev = pgstev << 24;
2454 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
2455 set_pgste_bits(kvm->mm, hva, mask, pgstev);
2456 }
2457 srcu_read_unlock(&kvm->srcu, srcu_idx);
2458 mmap_read_unlock(kvm->mm);
2459
2460 if (!kvm->mm->context.uses_cmm) {
2461 mmap_write_lock(kvm->mm);
2462 kvm->mm->context.uses_cmm = 1;
2463 mmap_write_unlock(kvm->mm);
2464 }
2465 out:
2466 vfree(bits);
2467 return r;
2468 }
2469
2470 /**
2471 * kvm_s390_cpus_from_pv - Convert all protected vCPUs in a protected VM to
2472 * non protected.
2473 * @kvm: the VM whose protected vCPUs are to be converted
2474 * @rc: return value for the RC field of the UVC (in case of error)
2475 * @rrc: return value for the RRC field of the UVC (in case of error)
2476 *
2477 * Does not stop in case of error, tries to convert as many
2478 * CPUs as possible. In case of error, the RC and RRC of the last error are
2479 * returned.
2480 *
2481 * Return: 0 in case of success, otherwise -EIO
2482 */
kvm_s390_cpus_from_pv(struct kvm * kvm,u16 * rc,u16 * rrc)2483 int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2484 {
2485 struct kvm_vcpu *vcpu;
2486 unsigned long i;
2487 u16 _rc, _rrc;
2488 int ret = 0;
2489
2490 /*
2491 * We ignore failures and try to destroy as many CPUs as possible.
2492 * At the same time we must not free the assigned resources when
2493 * this fails, as the ultravisor has still access to that memory.
2494 * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2495 * behind.
2496 * We want to return the first failure rc and rrc, though.
2497 */
2498 kvm_for_each_vcpu(i, vcpu, kvm) {
2499 mutex_lock(&vcpu->mutex);
2500 if (kvm_s390_pv_destroy_cpu(vcpu, &_rc, &_rrc) && !ret) {
2501 *rc = _rc;
2502 *rrc = _rrc;
2503 ret = -EIO;
2504 }
2505 mutex_unlock(&vcpu->mutex);
2506 }
2507 /* Ensure that we re-enable gisa if the non-PV guest used it but the PV guest did not. */
2508 if (use_gisa)
2509 kvm_s390_gisa_enable(kvm);
2510 return ret;
2511 }
2512
2513 /**
2514 * kvm_s390_cpus_to_pv - Convert all non-protected vCPUs in a protected VM
2515 * to protected.
2516 * @kvm: the VM whose protected vCPUs are to be converted
2517 * @rc: return value for the RC field of the UVC (in case of error)
2518 * @rrc: return value for the RRC field of the UVC (in case of error)
2519 *
2520 * Tries to undo the conversion in case of error.
2521 *
2522 * Return: 0 in case of success, otherwise -EIO
2523 */
kvm_s390_cpus_to_pv(struct kvm * kvm,u16 * rc,u16 * rrc)2524 static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2525 {
2526 unsigned long i;
2527 int r = 0;
2528 u16 dummy;
2529
2530 struct kvm_vcpu *vcpu;
2531
2532 /* Disable the GISA if the ultravisor does not support AIV. */
2533 if (!uv_has_feature(BIT_UV_FEAT_AIV))
2534 kvm_s390_gisa_disable(kvm);
2535
2536 kvm_for_each_vcpu(i, vcpu, kvm) {
2537 mutex_lock(&vcpu->mutex);
2538 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2539 mutex_unlock(&vcpu->mutex);
2540 if (r)
2541 break;
2542 }
2543 if (r)
2544 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2545 return r;
2546 }
2547
2548 /*
2549 * Here we provide user space with a direct interface to query UV
2550 * related data like UV maxima and available features as well as
2551 * feature specific data.
2552 *
2553 * To facilitate future extension of the data structures we'll try to
2554 * write data up to the maximum requested length.
2555 */
kvm_s390_handle_pv_info(struct kvm_s390_pv_info * info)2556 static ssize_t kvm_s390_handle_pv_info(struct kvm_s390_pv_info *info)
2557 {
2558 ssize_t len_min;
2559
2560 switch (info->header.id) {
2561 case KVM_PV_INFO_VM: {
2562 len_min = sizeof(info->header) + sizeof(info->vm);
2563
2564 if (info->header.len_max < len_min)
2565 return -EINVAL;
2566
2567 memcpy(info->vm.inst_calls_list,
2568 uv_info.inst_calls_list,
2569 sizeof(uv_info.inst_calls_list));
2570
2571 /* It's max cpuid not max cpus, so it's off by one */
2572 info->vm.max_cpus = uv_info.max_guest_cpu_id + 1;
2573 info->vm.max_guests = uv_info.max_num_sec_conf;
2574 info->vm.max_guest_addr = uv_info.max_sec_stor_addr;
2575 info->vm.feature_indication = uv_info.uv_feature_indications;
2576
2577 return len_min;
2578 }
2579 case KVM_PV_INFO_DUMP: {
2580 len_min = sizeof(info->header) + sizeof(info->dump);
2581
2582 if (info->header.len_max < len_min)
2583 return -EINVAL;
2584
2585 info->dump.dump_cpu_buffer_len = uv_info.guest_cpu_stor_len;
2586 info->dump.dump_config_mem_buffer_per_1m = uv_info.conf_dump_storage_state_len;
2587 info->dump.dump_config_finalize_len = uv_info.conf_dump_finalize_len;
2588 return len_min;
2589 }
2590 default:
2591 return -EINVAL;
2592 }
2593 }
2594
kvm_s390_pv_dmp(struct kvm * kvm,struct kvm_pv_cmd * cmd,struct kvm_s390_pv_dmp dmp)2595 static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd,
2596 struct kvm_s390_pv_dmp dmp)
2597 {
2598 int r = -EINVAL;
2599 void __user *result_buff = (void __user *)dmp.buff_addr;
2600
2601 switch (dmp.subcmd) {
2602 case KVM_PV_DUMP_INIT: {
2603 if (kvm->arch.pv.dumping)
2604 break;
2605
2606 /*
2607 * Block SIE entry as concurrent dump UVCs could lead
2608 * to validities.
2609 */
2610 kvm_s390_vcpu_block_all(kvm);
2611
2612 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2613 UVC_CMD_DUMP_INIT, &cmd->rc, &cmd->rrc);
2614 KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP INIT: rc %x rrc %x",
2615 cmd->rc, cmd->rrc);
2616 if (!r) {
2617 kvm->arch.pv.dumping = true;
2618 } else {
2619 kvm_s390_vcpu_unblock_all(kvm);
2620 r = -EINVAL;
2621 }
2622 break;
2623 }
2624 case KVM_PV_DUMP_CONFIG_STOR_STATE: {
2625 if (!kvm->arch.pv.dumping)
2626 break;
2627
2628 /*
2629 * gaddr is an output parameter since we might stop
2630 * early. As dmp will be copied back in our caller, we
2631 * don't need to do it ourselves.
2632 */
2633 r = kvm_s390_pv_dump_stor_state(kvm, result_buff, &dmp.gaddr, dmp.buff_len,
2634 &cmd->rc, &cmd->rrc);
2635 break;
2636 }
2637 case KVM_PV_DUMP_COMPLETE: {
2638 if (!kvm->arch.pv.dumping)
2639 break;
2640
2641 r = -EINVAL;
2642 if (dmp.buff_len < uv_info.conf_dump_finalize_len)
2643 break;
2644
2645 r = kvm_s390_pv_dump_complete(kvm, result_buff,
2646 &cmd->rc, &cmd->rrc);
2647 break;
2648 }
2649 default:
2650 r = -ENOTTY;
2651 break;
2652 }
2653
2654 return r;
2655 }
2656
kvm_s390_handle_pv(struct kvm * kvm,struct kvm_pv_cmd * cmd)2657 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2658 {
2659 const bool need_lock = (cmd->cmd != KVM_PV_ASYNC_CLEANUP_PERFORM);
2660 void __user *argp = (void __user *)cmd->data;
2661 int r = 0;
2662 u16 dummy;
2663
2664 if (need_lock)
2665 mutex_lock(&kvm->lock);
2666
2667 switch (cmd->cmd) {
2668 case KVM_PV_ENABLE: {
2669 r = -EINVAL;
2670 if (kvm_s390_pv_is_protected(kvm))
2671 break;
2672
2673 mmap_write_lock(kvm->mm);
2674 r = gmap_helper_disable_cow_sharing();
2675 mmap_write_unlock(kvm->mm);
2676 if (r)
2677 break;
2678
2679 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2680 if (r)
2681 break;
2682
2683 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2684 if (r)
2685 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
2686
2687 /* we need to block service interrupts from now on */
2688 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2689 break;
2690 }
2691 case KVM_PV_ASYNC_CLEANUP_PREPARE:
2692 r = -EINVAL;
2693 if (!kvm_s390_pv_is_protected(kvm) || !async_destroy)
2694 break;
2695
2696 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2697 /*
2698 * If a CPU could not be destroyed, destroy VM will also fail.
2699 * There is no point in trying to destroy it. Instead return
2700 * the rc and rrc from the first CPU that failed destroying.
2701 */
2702 if (r)
2703 break;
2704 r = kvm_s390_pv_set_aside(kvm, &cmd->rc, &cmd->rrc);
2705
2706 /* no need to block service interrupts any more */
2707 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2708 break;
2709 case KVM_PV_ASYNC_CLEANUP_PERFORM:
2710 r = -EINVAL;
2711 if (!async_destroy)
2712 break;
2713 /* kvm->lock must not be held; this is asserted inside the function. */
2714 r = kvm_s390_pv_deinit_aside_vm(kvm, &cmd->rc, &cmd->rrc);
2715 break;
2716 case KVM_PV_DISABLE: {
2717 r = -EINVAL;
2718 if (!kvm_s390_pv_is_protected(kvm))
2719 break;
2720
2721 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2722 /*
2723 * If a CPU could not be destroyed, destroy VM will also fail.
2724 * There is no point in trying to destroy it. Instead return
2725 * the rc and rrc from the first CPU that failed destroying.
2726 */
2727 if (r)
2728 break;
2729 r = kvm_s390_pv_deinit_cleanup_all(kvm, &cmd->rc, &cmd->rrc);
2730
2731 /* no need to block service interrupts any more */
2732 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2733 break;
2734 }
2735 case KVM_PV_SET_SEC_PARMS: {
2736 struct kvm_s390_pv_sec_parm parms = {};
2737 void *hdr;
2738
2739 r = -EINVAL;
2740 if (!kvm_s390_pv_is_protected(kvm))
2741 break;
2742
2743 r = -EFAULT;
2744 if (copy_from_user(&parms, argp, sizeof(parms)))
2745 break;
2746
2747 /* Currently restricted to 8KB */
2748 r = -EINVAL;
2749 if (parms.length > PAGE_SIZE * 2)
2750 break;
2751
2752 r = -ENOMEM;
2753 hdr = vmalloc(parms.length);
2754 if (!hdr)
2755 break;
2756
2757 r = -EFAULT;
2758 if (!copy_from_user(hdr, (void __user *)parms.origin,
2759 parms.length))
2760 r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2761 &cmd->rc, &cmd->rrc);
2762
2763 vfree(hdr);
2764 break;
2765 }
2766 case KVM_PV_UNPACK: {
2767 struct kvm_s390_pv_unp unp = {};
2768
2769 r = -EINVAL;
2770 if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
2771 break;
2772
2773 r = -EFAULT;
2774 if (copy_from_user(&unp, argp, sizeof(unp)))
2775 break;
2776
2777 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2778 &cmd->rc, &cmd->rrc);
2779 break;
2780 }
2781 case KVM_PV_VERIFY: {
2782 r = -EINVAL;
2783 if (!kvm_s390_pv_is_protected(kvm))
2784 break;
2785
2786 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2787 UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2788 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2789 cmd->rrc);
2790 break;
2791 }
2792 case KVM_PV_PREP_RESET: {
2793 r = -EINVAL;
2794 if (!kvm_s390_pv_is_protected(kvm))
2795 break;
2796
2797 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2798 UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2799 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2800 cmd->rc, cmd->rrc);
2801 break;
2802 }
2803 case KVM_PV_UNSHARE_ALL: {
2804 r = -EINVAL;
2805 if (!kvm_s390_pv_is_protected(kvm))
2806 break;
2807
2808 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2809 UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2810 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2811 cmd->rc, cmd->rrc);
2812 break;
2813 }
2814 case KVM_PV_INFO: {
2815 struct kvm_s390_pv_info info = {};
2816 ssize_t data_len;
2817
2818 /*
2819 * No need to check the VM protection here.
2820 *
2821 * Maybe user space wants to query some of the data
2822 * when the VM is still unprotected. If we see the
2823 * need to fence a new data command we can still
2824 * return an error in the info handler.
2825 */
2826
2827 r = -EFAULT;
2828 if (copy_from_user(&info, argp, sizeof(info.header)))
2829 break;
2830
2831 r = -EINVAL;
2832 if (info.header.len_max < sizeof(info.header))
2833 break;
2834
2835 data_len = kvm_s390_handle_pv_info(&info);
2836 if (data_len < 0) {
2837 r = data_len;
2838 break;
2839 }
2840 /*
2841 * If a data command struct is extended (multiple
2842 * times) this can be used to determine how much of it
2843 * is valid.
2844 */
2845 info.header.len_written = data_len;
2846
2847 r = -EFAULT;
2848 if (copy_to_user(argp, &info, data_len))
2849 break;
2850
2851 r = 0;
2852 break;
2853 }
2854 case KVM_PV_DUMP: {
2855 struct kvm_s390_pv_dmp dmp;
2856
2857 r = -EINVAL;
2858 if (!kvm_s390_pv_is_protected(kvm))
2859 break;
2860
2861 r = -EFAULT;
2862 if (copy_from_user(&dmp, argp, sizeof(dmp)))
2863 break;
2864
2865 r = kvm_s390_pv_dmp(kvm, cmd, dmp);
2866 if (r)
2867 break;
2868
2869 if (copy_to_user(argp, &dmp, sizeof(dmp))) {
2870 r = -EFAULT;
2871 break;
2872 }
2873
2874 break;
2875 }
2876 default:
2877 r = -ENOTTY;
2878 }
2879 if (need_lock)
2880 mutex_unlock(&kvm->lock);
2881
2882 return r;
2883 }
2884
mem_op_validate_common(struct kvm_s390_mem_op * mop,u64 supported_flags)2885 static int mem_op_validate_common(struct kvm_s390_mem_op *mop, u64 supported_flags)
2886 {
2887 if (mop->flags & ~supported_flags || !mop->size)
2888 return -EINVAL;
2889 if (mop->size > MEM_OP_MAX_SIZE)
2890 return -E2BIG;
2891 if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
2892 if (mop->key > 0xf)
2893 return -EINVAL;
2894 } else {
2895 mop->key = 0;
2896 }
2897 return 0;
2898 }
2899
kvm_s390_vm_mem_op_abs(struct kvm * kvm,struct kvm_s390_mem_op * mop)2900 static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2901 {
2902 void __user *uaddr = (void __user *)mop->buf;
2903 enum gacc_mode acc_mode;
2904 void *tmpbuf = NULL;
2905 int r, srcu_idx;
2906
2907 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION |
2908 KVM_S390_MEMOP_F_CHECK_ONLY);
2909 if (r)
2910 return r;
2911
2912 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2913 tmpbuf = vmalloc(mop->size);
2914 if (!tmpbuf)
2915 return -ENOMEM;
2916 }
2917
2918 srcu_idx = srcu_read_lock(&kvm->srcu);
2919
2920 if (!kvm_is_gpa_in_memslot(kvm, mop->gaddr)) {
2921 r = PGM_ADDRESSING;
2922 goto out_unlock;
2923 }
2924
2925 acc_mode = mop->op == KVM_S390_MEMOP_ABSOLUTE_READ ? GACC_FETCH : GACC_STORE;
2926 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
2927 r = check_gpa_range(kvm, mop->gaddr, mop->size, acc_mode, mop->key);
2928 goto out_unlock;
2929 }
2930 if (acc_mode == GACC_FETCH) {
2931 r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
2932 mop->size, GACC_FETCH, mop->key);
2933 if (r)
2934 goto out_unlock;
2935 if (copy_to_user(uaddr, tmpbuf, mop->size))
2936 r = -EFAULT;
2937 } else {
2938 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
2939 r = -EFAULT;
2940 goto out_unlock;
2941 }
2942 r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
2943 mop->size, GACC_STORE, mop->key);
2944 }
2945
2946 out_unlock:
2947 srcu_read_unlock(&kvm->srcu, srcu_idx);
2948
2949 vfree(tmpbuf);
2950 return r;
2951 }
2952
kvm_s390_vm_mem_op_cmpxchg(struct kvm * kvm,struct kvm_s390_mem_op * mop)2953 static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2954 {
2955 void __user *uaddr = (void __user *)mop->buf;
2956 void __user *old_addr = (void __user *)mop->old_addr;
2957 union {
2958 __uint128_t quad;
2959 char raw[sizeof(__uint128_t)];
2960 } old = { .quad = 0}, new = { .quad = 0 };
2961 unsigned int off_in_quad = sizeof(new) - mop->size;
2962 int r, srcu_idx;
2963 bool success;
2964
2965 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION);
2966 if (r)
2967 return r;
2968 /*
2969 * This validates off_in_quad. Checking that size is a power
2970 * of two is not necessary, as cmpxchg_guest_abs_with_key
2971 * takes care of that
2972 */
2973 if (mop->size > sizeof(new))
2974 return -EINVAL;
2975 if (copy_from_user(&new.raw[off_in_quad], uaddr, mop->size))
2976 return -EFAULT;
2977 if (copy_from_user(&old.raw[off_in_quad], old_addr, mop->size))
2978 return -EFAULT;
2979
2980 srcu_idx = srcu_read_lock(&kvm->srcu);
2981
2982 if (!kvm_is_gpa_in_memslot(kvm, mop->gaddr)) {
2983 r = PGM_ADDRESSING;
2984 goto out_unlock;
2985 }
2986
2987 r = cmpxchg_guest_abs_with_key(kvm, mop->gaddr, mop->size, &old.quad,
2988 new.quad, mop->key, &success);
2989 if (!success && copy_to_user(old_addr, &old.raw[off_in_quad], mop->size))
2990 r = -EFAULT;
2991
2992 out_unlock:
2993 srcu_read_unlock(&kvm->srcu, srcu_idx);
2994 return r;
2995 }
2996
kvm_s390_vm_mem_op(struct kvm * kvm,struct kvm_s390_mem_op * mop)2997 static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2998 {
2999 /*
3000 * This is technically a heuristic only, if the kvm->lock is not
3001 * taken, it is not guaranteed that the vm is/remains non-protected.
3002 * This is ok from a kernel perspective, wrongdoing is detected
3003 * on the access, -EFAULT is returned and the vm may crash the
3004 * next time it accesses the memory in question.
3005 * There is no sane usecase to do switching and a memop on two
3006 * different CPUs at the same time.
3007 */
3008 if (kvm_s390_pv_get_handle(kvm))
3009 return -EINVAL;
3010
3011 switch (mop->op) {
3012 case KVM_S390_MEMOP_ABSOLUTE_READ:
3013 case KVM_S390_MEMOP_ABSOLUTE_WRITE:
3014 return kvm_s390_vm_mem_op_abs(kvm, mop);
3015 case KVM_S390_MEMOP_ABSOLUTE_CMPXCHG:
3016 return kvm_s390_vm_mem_op_cmpxchg(kvm, mop);
3017 default:
3018 return -EINVAL;
3019 }
3020 }
3021
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)3022 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
3023 {
3024 struct kvm *kvm = filp->private_data;
3025 void __user *argp = (void __user *)arg;
3026 struct kvm_device_attr attr;
3027 int r;
3028
3029 switch (ioctl) {
3030 case KVM_S390_INTERRUPT: {
3031 struct kvm_s390_interrupt s390int;
3032
3033 r = -EFAULT;
3034 if (copy_from_user(&s390int, argp, sizeof(s390int)))
3035 break;
3036 r = kvm_s390_inject_vm(kvm, &s390int);
3037 break;
3038 }
3039 case KVM_CREATE_IRQCHIP: {
3040 r = -EINVAL;
3041 if (kvm->arch.use_irqchip)
3042 r = 0;
3043 break;
3044 }
3045 case KVM_SET_DEVICE_ATTR: {
3046 r = -EFAULT;
3047 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
3048 break;
3049 r = kvm_s390_vm_set_attr(kvm, &attr);
3050 break;
3051 }
3052 case KVM_GET_DEVICE_ATTR: {
3053 r = -EFAULT;
3054 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
3055 break;
3056 r = kvm_s390_vm_get_attr(kvm, &attr);
3057 break;
3058 }
3059 case KVM_HAS_DEVICE_ATTR: {
3060 r = -EFAULT;
3061 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
3062 break;
3063 r = kvm_s390_vm_has_attr(kvm, &attr);
3064 break;
3065 }
3066 case KVM_S390_GET_SKEYS: {
3067 struct kvm_s390_skeys args;
3068
3069 r = -EFAULT;
3070 if (copy_from_user(&args, argp,
3071 sizeof(struct kvm_s390_skeys)))
3072 break;
3073 r = kvm_s390_get_skeys(kvm, &args);
3074 break;
3075 }
3076 case KVM_S390_SET_SKEYS: {
3077 struct kvm_s390_skeys args;
3078
3079 r = -EFAULT;
3080 if (copy_from_user(&args, argp,
3081 sizeof(struct kvm_s390_skeys)))
3082 break;
3083 r = kvm_s390_set_skeys(kvm, &args);
3084 break;
3085 }
3086 case KVM_S390_GET_CMMA_BITS: {
3087 struct kvm_s390_cmma_log args;
3088
3089 r = -EFAULT;
3090 if (copy_from_user(&args, argp, sizeof(args)))
3091 break;
3092 mutex_lock(&kvm->slots_lock);
3093 r = kvm_s390_get_cmma_bits(kvm, &args);
3094 mutex_unlock(&kvm->slots_lock);
3095 if (!r) {
3096 r = copy_to_user(argp, &args, sizeof(args));
3097 if (r)
3098 r = -EFAULT;
3099 }
3100 break;
3101 }
3102 case KVM_S390_SET_CMMA_BITS: {
3103 struct kvm_s390_cmma_log args;
3104
3105 r = -EFAULT;
3106 if (copy_from_user(&args, argp, sizeof(args)))
3107 break;
3108 mutex_lock(&kvm->slots_lock);
3109 r = kvm_s390_set_cmma_bits(kvm, &args);
3110 mutex_unlock(&kvm->slots_lock);
3111 break;
3112 }
3113 case KVM_S390_PV_COMMAND: {
3114 struct kvm_pv_cmd args;
3115
3116 /* protvirt means user cpu state */
3117 kvm_s390_set_user_cpu_state_ctrl(kvm);
3118 r = 0;
3119 if (!is_prot_virt_host()) {
3120 r = -EINVAL;
3121 break;
3122 }
3123 if (copy_from_user(&args, argp, sizeof(args))) {
3124 r = -EFAULT;
3125 break;
3126 }
3127 if (args.flags) {
3128 r = -EINVAL;
3129 break;
3130 }
3131 /* must be called without kvm->lock */
3132 r = kvm_s390_handle_pv(kvm, &args);
3133 if (copy_to_user(argp, &args, sizeof(args))) {
3134 r = -EFAULT;
3135 break;
3136 }
3137 break;
3138 }
3139 case KVM_S390_MEM_OP: {
3140 struct kvm_s390_mem_op mem_op;
3141
3142 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3143 r = kvm_s390_vm_mem_op(kvm, &mem_op);
3144 else
3145 r = -EFAULT;
3146 break;
3147 }
3148 case KVM_S390_ZPCI_OP: {
3149 struct kvm_s390_zpci_op args;
3150
3151 r = -EINVAL;
3152 if (!IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3153 break;
3154 if (copy_from_user(&args, argp, sizeof(args))) {
3155 r = -EFAULT;
3156 break;
3157 }
3158 r = kvm_s390_pci_zpci_op(kvm, &args);
3159 break;
3160 }
3161 default:
3162 r = -ENOTTY;
3163 }
3164
3165 return r;
3166 }
3167
kvm_s390_apxa_installed(void)3168 static int kvm_s390_apxa_installed(void)
3169 {
3170 struct ap_config_info info;
3171
3172 if (ap_instructions_available()) {
3173 if (ap_qci(&info) == 0)
3174 return info.apxa;
3175 }
3176
3177 return 0;
3178 }
3179
3180 /*
3181 * The format of the crypto control block (CRYCB) is specified in the 3 low
3182 * order bits of the CRYCB designation (CRYCBD) field as follows:
3183 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
3184 * AP extended addressing (APXA) facility are installed.
3185 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
3186 * Format 2: Both the APXA and MSAX3 facilities are installed
3187 */
kvm_s390_set_crycb_format(struct kvm * kvm)3188 static void kvm_s390_set_crycb_format(struct kvm *kvm)
3189 {
3190 kvm->arch.crypto.crycbd = virt_to_phys(kvm->arch.crypto.crycb);
3191
3192 /* Clear the CRYCB format bits - i.e., set format 0 by default */
3193 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
3194
3195 /* Check whether MSAX3 is installed */
3196 if (!test_kvm_facility(kvm, 76))
3197 return;
3198
3199 if (kvm_s390_apxa_installed())
3200 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
3201 else
3202 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
3203 }
3204
3205 /*
3206 * kvm_arch_crypto_set_masks
3207 *
3208 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
3209 * to be set.
3210 * @apm: the mask identifying the accessible AP adapters
3211 * @aqm: the mask identifying the accessible AP domains
3212 * @adm: the mask identifying the accessible AP control domains
3213 *
3214 * Set the masks that identify the adapters, domains and control domains to
3215 * which the KVM guest is granted access.
3216 *
3217 * Note: The kvm->lock mutex must be locked by the caller before invoking this
3218 * function.
3219 */
kvm_arch_crypto_set_masks(struct kvm * kvm,unsigned long * apm,unsigned long * aqm,unsigned long * adm)3220 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
3221 unsigned long *aqm, unsigned long *adm)
3222 {
3223 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
3224
3225 kvm_s390_vcpu_block_all(kvm);
3226
3227 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
3228 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
3229 memcpy(crycb->apcb1.apm, apm, 32);
3230 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
3231 apm[0], apm[1], apm[2], apm[3]);
3232 memcpy(crycb->apcb1.aqm, aqm, 32);
3233 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
3234 aqm[0], aqm[1], aqm[2], aqm[3]);
3235 memcpy(crycb->apcb1.adm, adm, 32);
3236 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
3237 adm[0], adm[1], adm[2], adm[3]);
3238 break;
3239 case CRYCB_FORMAT1:
3240 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
3241 memcpy(crycb->apcb0.apm, apm, 8);
3242 memcpy(crycb->apcb0.aqm, aqm, 2);
3243 memcpy(crycb->apcb0.adm, adm, 2);
3244 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
3245 apm[0], *((unsigned short *)aqm),
3246 *((unsigned short *)adm));
3247 break;
3248 default: /* Can not happen */
3249 break;
3250 }
3251
3252 /* recreate the shadow crycb for each vcpu */
3253 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3254 kvm_s390_vcpu_unblock_all(kvm);
3255 }
3256 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
3257
3258 /*
3259 * kvm_arch_crypto_clear_masks
3260 *
3261 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
3262 * to be cleared.
3263 *
3264 * Clear the masks that identify the adapters, domains and control domains to
3265 * which the KVM guest is granted access.
3266 *
3267 * Note: The kvm->lock mutex must be locked by the caller before invoking this
3268 * function.
3269 */
kvm_arch_crypto_clear_masks(struct kvm * kvm)3270 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
3271 {
3272 kvm_s390_vcpu_block_all(kvm);
3273
3274 memset(&kvm->arch.crypto.crycb->apcb0, 0,
3275 sizeof(kvm->arch.crypto.crycb->apcb0));
3276 memset(&kvm->arch.crypto.crycb->apcb1, 0,
3277 sizeof(kvm->arch.crypto.crycb->apcb1));
3278
3279 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
3280 /* recreate the shadow crycb for each vcpu */
3281 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3282 kvm_s390_vcpu_unblock_all(kvm);
3283 }
3284 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
3285
kvm_s390_get_initial_cpuid(void)3286 static u64 kvm_s390_get_initial_cpuid(void)
3287 {
3288 struct cpuid cpuid;
3289
3290 get_cpu_id(&cpuid);
3291 cpuid.version = 0xff;
3292 return *((u64 *) &cpuid);
3293 }
3294
kvm_s390_crypto_init(struct kvm * kvm)3295 static void kvm_s390_crypto_init(struct kvm *kvm)
3296 {
3297 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
3298 kvm_s390_set_crycb_format(kvm);
3299 init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
3300
3301 if (!test_kvm_facility(kvm, 76))
3302 return;
3303
3304 /* Enable AES/DEA protected key functions by default */
3305 kvm->arch.crypto.aes_kw = 1;
3306 kvm->arch.crypto.dea_kw = 1;
3307 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
3308 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
3309 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
3310 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
3311 }
3312
sca_dispose(struct kvm * kvm)3313 static void sca_dispose(struct kvm *kvm)
3314 {
3315 free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca));
3316 kvm->arch.sca = NULL;
3317 }
3318
kvm_arch_free_vm(struct kvm * kvm)3319 void kvm_arch_free_vm(struct kvm *kvm)
3320 {
3321 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3322 kvm_s390_pci_clear_list(kvm);
3323
3324 __kvm_arch_free_vm(kvm);
3325 }
3326
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)3327 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
3328 {
3329 gfp_t alloc_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
3330 char debug_name[16];
3331 int i, rc;
3332
3333 rc = -EINVAL;
3334 #ifdef CONFIG_KVM_S390_UCONTROL
3335 if (type & ~KVM_VM_S390_UCONTROL)
3336 goto out_err;
3337 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
3338 goto out_err;
3339 #else
3340 if (type)
3341 goto out_err;
3342 #endif
3343
3344 rc = s390_enable_sie();
3345 if (rc)
3346 goto out_err;
3347
3348 rc = -ENOMEM;
3349
3350 if (!sclp.has_64bscao)
3351 alloc_flags |= GFP_DMA;
3352 mutex_lock(&kvm_lock);
3353
3354 kvm->arch.sca = alloc_pages_exact(sizeof(*kvm->arch.sca), alloc_flags);
3355 mutex_unlock(&kvm_lock);
3356 if (!kvm->arch.sca)
3357 goto out_err;
3358
3359 snprintf(debug_name, sizeof(debug_name), "kvm-%u", current->pid);
3360
3361 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
3362 if (!kvm->arch.dbf)
3363 goto out_err;
3364
3365 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
3366 kvm->arch.sie_page2 =
3367 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3368 if (!kvm->arch.sie_page2)
3369 goto out_err;
3370
3371 kvm->arch.sie_page2->kvm = kvm;
3372 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
3373
3374 for (i = 0; i < kvm_s390_fac_size(); i++) {
3375 kvm->arch.model.fac_mask[i] = stfle_fac_list[i] &
3376 (kvm_s390_fac_base[i] |
3377 kvm_s390_fac_ext[i]);
3378 kvm->arch.model.fac_list[i] = stfle_fac_list[i] &
3379 kvm_s390_fac_base[i];
3380 }
3381 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
3382
3383 /* we are always in czam mode - even on pre z14 machines */
3384 set_kvm_facility(kvm->arch.model.fac_mask, 138);
3385 set_kvm_facility(kvm->arch.model.fac_list, 138);
3386 /* we emulate STHYI in kvm */
3387 set_kvm_facility(kvm->arch.model.fac_mask, 74);
3388 set_kvm_facility(kvm->arch.model.fac_list, 74);
3389 if (machine_has_tlb_guest()) {
3390 set_kvm_facility(kvm->arch.model.fac_mask, 147);
3391 set_kvm_facility(kvm->arch.model.fac_list, 147);
3392 }
3393
3394 if (css_general_characteristics.aiv && test_facility(65))
3395 set_kvm_facility(kvm->arch.model.fac_mask, 65);
3396
3397 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
3398 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
3399
3400 kvm->arch.model.uv_feat_guest.feat = 0;
3401
3402 kvm_s390_crypto_init(kvm);
3403
3404 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3405 mutex_lock(&kvm->lock);
3406 kvm_s390_pci_init_list(kvm);
3407 kvm_s390_vcpu_pci_enable_interp(kvm);
3408 mutex_unlock(&kvm->lock);
3409 }
3410
3411 mutex_init(&kvm->arch.float_int.ais_lock);
3412 spin_lock_init(&kvm->arch.float_int.lock);
3413 for (i = 0; i < FIRQ_LIST_COUNT; i++)
3414 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
3415 init_waitqueue_head(&kvm->arch.ipte_wq);
3416 mutex_init(&kvm->arch.ipte_mutex);
3417
3418 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
3419 VM_EVENT(kvm, 3, "vm created with type %lu", type);
3420
3421 if (type & KVM_VM_S390_UCONTROL) {
3422 struct kvm_userspace_memory_region2 fake_memslot = {
3423 .slot = KVM_S390_UCONTROL_MEMSLOT,
3424 .guest_phys_addr = 0,
3425 .userspace_addr = 0,
3426 .memory_size = ALIGN_DOWN(TASK_SIZE, _SEGMENT_SIZE),
3427 .flags = 0,
3428 };
3429
3430 kvm->arch.gmap = NULL;
3431 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
3432 /* one flat fake memslot covering the whole address-space */
3433 mutex_lock(&kvm->slots_lock);
3434 KVM_BUG_ON(kvm_set_internal_memslot(kvm, &fake_memslot), kvm);
3435 mutex_unlock(&kvm->slots_lock);
3436 } else {
3437 if (sclp.hamax == U64_MAX)
3438 kvm->arch.mem_limit = TASK_SIZE_MAX;
3439 else
3440 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
3441 sclp.hamax + 1);
3442 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
3443 if (!kvm->arch.gmap)
3444 goto out_err;
3445 kvm->arch.gmap->private = kvm;
3446 kvm->arch.gmap->pfault_enabled = 0;
3447 }
3448
3449 kvm->arch.use_pfmfi = sclp.has_pfmfi;
3450 kvm->arch.use_skf = sclp.has_skey;
3451 spin_lock_init(&kvm->arch.start_stop_lock);
3452 kvm_s390_vsie_init(kvm);
3453 if (use_gisa)
3454 kvm_s390_gisa_init(kvm);
3455 INIT_LIST_HEAD(&kvm->arch.pv.need_cleanup);
3456 kvm->arch.pv.set_aside = NULL;
3457 KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
3458
3459 return 0;
3460 out_err:
3461 free_page((unsigned long)kvm->arch.sie_page2);
3462 debug_unregister(kvm->arch.dbf);
3463 sca_dispose(kvm);
3464 KVM_EVENT(3, "creation of vm failed: %d", rc);
3465 return rc;
3466 }
3467
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)3468 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
3469 {
3470 u16 rc, rrc;
3471
3472 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
3473 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
3474 kvm_s390_clear_local_irqs(vcpu);
3475 kvm_clear_async_pf_completion_queue(vcpu);
3476 if (!kvm_is_ucontrol(vcpu->kvm))
3477 sca_del_vcpu(vcpu);
3478 kvm_s390_update_topology_change_report(vcpu->kvm, 1);
3479
3480 if (kvm_is_ucontrol(vcpu->kvm))
3481 gmap_remove(vcpu->arch.gmap);
3482
3483 if (vcpu->kvm->arch.use_cmma)
3484 kvm_s390_vcpu_unsetup_cmma(vcpu);
3485 /* We can not hold the vcpu mutex here, we are already dying */
3486 if (kvm_s390_pv_cpu_get_handle(vcpu))
3487 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
3488 free_page((unsigned long)(vcpu->arch.sie_block));
3489 }
3490
kvm_arch_destroy_vm(struct kvm * kvm)3491 void kvm_arch_destroy_vm(struct kvm *kvm)
3492 {
3493 u16 rc, rrc;
3494
3495 kvm_destroy_vcpus(kvm);
3496 sca_dispose(kvm);
3497 kvm_s390_gisa_destroy(kvm);
3498 /*
3499 * We are already at the end of life and kvm->lock is not taken.
3500 * This is ok as the file descriptor is closed by now and nobody
3501 * can mess with the pv state.
3502 */
3503 kvm_s390_pv_deinit_cleanup_all(kvm, &rc, &rrc);
3504 /*
3505 * Remove the mmu notifier only when the whole KVM VM is torn down,
3506 * and only if one was registered to begin with. If the VM is
3507 * currently not protected, but has been previously been protected,
3508 * then it's possible that the notifier is still registered.
3509 */
3510 if (kvm->arch.pv.mmu_notifier.ops)
3511 mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm);
3512
3513 debug_unregister(kvm->arch.dbf);
3514 free_page((unsigned long)kvm->arch.sie_page2);
3515 if (!kvm_is_ucontrol(kvm))
3516 gmap_remove(kvm->arch.gmap);
3517 kvm_s390_destroy_adapters(kvm);
3518 kvm_s390_clear_float_irqs(kvm);
3519 kvm_s390_vsie_destroy(kvm);
3520 KVM_EVENT(3, "vm 0x%p destroyed", kvm);
3521 }
3522
3523 /* Section: vcpu related */
__kvm_ucontrol_vcpu_init(struct kvm_vcpu * vcpu)3524 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
3525 {
3526 vcpu->arch.gmap = gmap_create(current->mm, -1UL);
3527 if (!vcpu->arch.gmap)
3528 return -ENOMEM;
3529 vcpu->arch.gmap->private = vcpu->kvm;
3530
3531 return 0;
3532 }
3533
sca_del_vcpu(struct kvm_vcpu * vcpu)3534 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
3535 {
3536 struct esca_block *sca = vcpu->kvm->arch.sca;
3537
3538 if (!kvm_s390_use_sca_entries())
3539 return;
3540
3541 clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3542 sca->cpu[vcpu->vcpu_id].sda = 0;
3543 }
3544
sca_add_vcpu(struct kvm_vcpu * vcpu)3545 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
3546 {
3547 struct esca_block *sca = vcpu->kvm->arch.sca;
3548 phys_addr_t sca_phys = virt_to_phys(sca);
3549
3550 /* we still need the sca header for the ipte control */
3551 vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3552 vcpu->arch.sie_block->scaol = sca_phys & ESCA_SCAOL_MASK;
3553 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
3554
3555 if (!kvm_s390_use_sca_entries())
3556 return;
3557
3558 set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3559 sca->cpu[vcpu->vcpu_id].sda = virt_to_phys(vcpu->arch.sie_block);
3560 }
3561
sca_can_add_vcpu(struct kvm * kvm,unsigned int id)3562 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
3563 {
3564 if (!kvm_s390_use_sca_entries())
3565 return id < KVM_MAX_VCPUS;
3566
3567 return id < KVM_S390_ESCA_CPU_SLOTS;
3568 }
3569
3570 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__start_cpu_timer_accounting(struct kvm_vcpu * vcpu)3571 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3572 {
3573 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
3574 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3575 vcpu->arch.cputm_start = get_tod_clock_fast();
3576 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3577 }
3578
3579 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__stop_cpu_timer_accounting(struct kvm_vcpu * vcpu)3580 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3581 {
3582 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
3583 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3584 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3585 vcpu->arch.cputm_start = 0;
3586 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3587 }
3588
3589 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__enable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3590 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3591 {
3592 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
3593 vcpu->arch.cputm_enabled = true;
3594 __start_cpu_timer_accounting(vcpu);
3595 }
3596
3597 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__disable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3598 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3599 {
3600 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
3601 __stop_cpu_timer_accounting(vcpu);
3602 vcpu->arch.cputm_enabled = false;
3603 }
3604
enable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3605 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3606 {
3607 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3608 __enable_cpu_timer_accounting(vcpu);
3609 preempt_enable();
3610 }
3611
disable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3612 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3613 {
3614 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3615 __disable_cpu_timer_accounting(vcpu);
3616 preempt_enable();
3617 }
3618
3619 /* set the cpu timer - may only be called from the VCPU thread itself */
kvm_s390_set_cpu_timer(struct kvm_vcpu * vcpu,__u64 cputm)3620 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3621 {
3622 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3623 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3624 if (vcpu->arch.cputm_enabled)
3625 vcpu->arch.cputm_start = get_tod_clock_fast();
3626 vcpu->arch.sie_block->cputm = cputm;
3627 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3628 preempt_enable();
3629 }
3630
3631 /* update and get the cpu timer - can also be called from other VCPU threads */
kvm_s390_get_cpu_timer(struct kvm_vcpu * vcpu)3632 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3633 {
3634 unsigned int seq;
3635 __u64 value;
3636
3637 if (unlikely(!vcpu->arch.cputm_enabled))
3638 return vcpu->arch.sie_block->cputm;
3639
3640 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3641 do {
3642 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3643 /*
3644 * If the writer would ever execute a read in the critical
3645 * section, e.g. in irq context, we have a deadlock.
3646 */
3647 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3648 value = vcpu->arch.sie_block->cputm;
3649 /* if cputm_start is 0, accounting is being started/stopped */
3650 if (likely(vcpu->arch.cputm_start))
3651 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3652 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3653 preempt_enable();
3654 return value;
3655 }
3656
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)3657 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3658 {
3659
3660 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
3661 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3662 __start_cpu_timer_accounting(vcpu);
3663 vcpu->cpu = cpu;
3664 }
3665
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)3666 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3667 {
3668 vcpu->cpu = -1;
3669 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3670 __stop_cpu_timer_accounting(vcpu);
3671 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
3672
3673 }
3674
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)3675 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
3676 {
3677 mutex_lock(&vcpu->kvm->lock);
3678 preempt_disable();
3679 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
3680 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
3681 preempt_enable();
3682 mutex_unlock(&vcpu->kvm->lock);
3683 if (!kvm_is_ucontrol(vcpu->kvm)) {
3684 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
3685 sca_add_vcpu(vcpu);
3686 }
3687 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
3688 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3689 }
3690
kvm_has_pckmo_subfunc(struct kvm * kvm,unsigned long nr)3691 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3692 {
3693 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3694 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3695 return true;
3696 return false;
3697 }
3698
kvm_has_pckmo_ecc(struct kvm * kvm)3699 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3700 {
3701 /* At least one ECC subfunction must be present */
3702 return kvm_has_pckmo_subfunc(kvm, 32) ||
3703 kvm_has_pckmo_subfunc(kvm, 33) ||
3704 kvm_has_pckmo_subfunc(kvm, 34) ||
3705 kvm_has_pckmo_subfunc(kvm, 40) ||
3706 kvm_has_pckmo_subfunc(kvm, 41);
3707
3708 }
3709
kvm_has_pckmo_hmac(struct kvm * kvm)3710 static bool kvm_has_pckmo_hmac(struct kvm *kvm)
3711 {
3712 /* At least one HMAC subfunction must be present */
3713 return kvm_has_pckmo_subfunc(kvm, 118) ||
3714 kvm_has_pckmo_subfunc(kvm, 122);
3715 }
3716
kvm_s390_vcpu_crypto_setup(struct kvm_vcpu * vcpu)3717 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3718 {
3719 /*
3720 * If the AP instructions are not being interpreted and the MSAX3
3721 * facility is not configured for the guest, there is nothing to set up.
3722 */
3723 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
3724 return;
3725
3726 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
3727 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
3728 vcpu->arch.sie_block->eca &= ~ECA_APIE;
3729 vcpu->arch.sie_block->ecd &= ~(ECD_ECC | ECD_HMAC);
3730
3731 if (vcpu->kvm->arch.crypto.apie)
3732 vcpu->arch.sie_block->eca |= ECA_APIE;
3733
3734 /* Set up protected key support */
3735 if (vcpu->kvm->arch.crypto.aes_kw) {
3736 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
3737 /* ecc/hmac is also wrapped with AES key */
3738 if (kvm_has_pckmo_ecc(vcpu->kvm))
3739 vcpu->arch.sie_block->ecd |= ECD_ECC;
3740 if (kvm_has_pckmo_hmac(vcpu->kvm))
3741 vcpu->arch.sie_block->ecd |= ECD_HMAC;
3742 }
3743
3744 if (vcpu->kvm->arch.crypto.dea_kw)
3745 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
3746 }
3747
kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu * vcpu)3748 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3749 {
3750 free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo));
3751 vcpu->arch.sie_block->cbrlo = 0;
3752 }
3753
kvm_s390_vcpu_setup_cmma(struct kvm_vcpu * vcpu)3754 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3755 {
3756 void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3757
3758 if (!cbrlo_page)
3759 return -ENOMEM;
3760
3761 vcpu->arch.sie_block->cbrlo = virt_to_phys(cbrlo_page);
3762 return 0;
3763 }
3764
kvm_s390_vcpu_setup_model(struct kvm_vcpu * vcpu)3765 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3766 {
3767 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3768
3769 vcpu->arch.sie_block->ibc = model->ibc;
3770 if (test_kvm_facility(vcpu->kvm, 7))
3771 vcpu->arch.sie_block->fac = virt_to_phys(model->fac_list);
3772 }
3773
kvm_s390_vcpu_setup(struct kvm_vcpu * vcpu)3774 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
3775 {
3776 int rc = 0;
3777 u16 uvrc, uvrrc;
3778
3779 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3780 CPUSTAT_SM |
3781 CPUSTAT_STOPPED);
3782
3783 if (test_kvm_facility(vcpu->kvm, 78))
3784 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
3785 else if (test_kvm_facility(vcpu->kvm, 8))
3786 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
3787
3788 kvm_s390_vcpu_setup_model(vcpu);
3789
3790 /* pgste_set_pte has special handling for !machine_has_esop() */
3791 if (machine_has_esop())
3792 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
3793 if (test_kvm_facility(vcpu->kvm, 9))
3794 vcpu->arch.sie_block->ecb |= ECB_SRSI;
3795 if (test_kvm_facility(vcpu->kvm, 11))
3796 vcpu->arch.sie_block->ecb |= ECB_PTF;
3797 if (test_kvm_facility(vcpu->kvm, 73))
3798 vcpu->arch.sie_block->ecb |= ECB_TE;
3799 if (!kvm_is_ucontrol(vcpu->kvm))
3800 vcpu->arch.sie_block->ecb |= ECB_SPECI;
3801
3802 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
3803 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
3804 if (test_kvm_facility(vcpu->kvm, 130))
3805 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3806 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3807 if (sclp.has_cei)
3808 vcpu->arch.sie_block->eca |= ECA_CEI;
3809 if (sclp.has_ib)
3810 vcpu->arch.sie_block->eca |= ECA_IB;
3811 if (sclp.has_siif)
3812 vcpu->arch.sie_block->eca |= ECA_SII;
3813 if (kvm_s390_use_sca_entries())
3814 vcpu->arch.sie_block->eca |= ECA_SIGPI;
3815 if (test_kvm_facility(vcpu->kvm, 129)) {
3816 vcpu->arch.sie_block->eca |= ECA_VX;
3817 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3818 }
3819 if (test_kvm_facility(vcpu->kvm, 139))
3820 vcpu->arch.sie_block->ecd |= ECD_MEF;
3821 if (test_kvm_facility(vcpu->kvm, 156))
3822 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3823 if (vcpu->arch.sie_block->gd) {
3824 vcpu->arch.sie_block->eca |= ECA_AIV;
3825 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3826 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3827 }
3828 vcpu->arch.sie_block->sdnxo = virt_to_phys(&vcpu->run->s.regs.sdnx) | SDNXC;
3829 vcpu->arch.sie_block->riccbd = virt_to_phys(&vcpu->run->s.regs.riccb);
3830
3831 if (sclp.has_kss)
3832 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3833 else
3834 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3835
3836 if (vcpu->kvm->arch.use_cmma) {
3837 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3838 if (rc)
3839 return rc;
3840 }
3841 hrtimer_setup(&vcpu->arch.ckc_timer, kvm_s390_idle_wakeup, CLOCK_MONOTONIC,
3842 HRTIMER_MODE_REL);
3843
3844 vcpu->arch.sie_block->hpid = HPID_KVM;
3845
3846 kvm_s390_vcpu_crypto_setup(vcpu);
3847
3848 kvm_s390_vcpu_pci_setup(vcpu);
3849
3850 mutex_lock(&vcpu->kvm->lock);
3851 if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3852 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3853 if (rc)
3854 kvm_s390_vcpu_unsetup_cmma(vcpu);
3855 }
3856 mutex_unlock(&vcpu->kvm->lock);
3857
3858 return rc;
3859 }
3860
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)3861 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3862 {
3863 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3864 return -EINVAL;
3865 return 0;
3866 }
3867
kvm_arch_vcpu_create(struct kvm_vcpu * vcpu)3868 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
3869 {
3870 struct sie_page *sie_page;
3871 int rc;
3872
3873 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3874 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
3875 if (!sie_page)
3876 return -ENOMEM;
3877
3878 vcpu->arch.sie_block = &sie_page->sie_block;
3879 vcpu->arch.sie_block->itdba = virt_to_phys(&sie_page->itdb);
3880
3881 /* the real guest size will always be smaller than msl */
3882 vcpu->arch.sie_block->mso = 0;
3883 vcpu->arch.sie_block->msl = sclp.hamax;
3884
3885 vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
3886 spin_lock_init(&vcpu->arch.local_int.lock);
3887 vcpu->arch.sie_block->gd = kvm_s390_get_gisa_desc(vcpu->kvm);
3888 seqcount_init(&vcpu->arch.cputm_seqcount);
3889
3890 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3891 kvm_clear_async_pf_completion_queue(vcpu);
3892 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3893 KVM_SYNC_GPRS |
3894 KVM_SYNC_ACRS |
3895 KVM_SYNC_CRS |
3896 KVM_SYNC_ARCH0 |
3897 KVM_SYNC_PFAULT |
3898 KVM_SYNC_DIAG318;
3899 vcpu->arch.acrs_loaded = false;
3900 kvm_s390_set_prefix(vcpu, 0);
3901 if (test_kvm_facility(vcpu->kvm, 64))
3902 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3903 if (test_kvm_facility(vcpu->kvm, 82))
3904 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3905 if (test_kvm_facility(vcpu->kvm, 133))
3906 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3907 if (test_kvm_facility(vcpu->kvm, 156))
3908 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3909 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3910 * cpu_has_vx(), (load|store)_fpu_regs() will work with vrs format.
3911 */
3912 if (cpu_has_vx())
3913 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3914 else
3915 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3916
3917 if (kvm_is_ucontrol(vcpu->kvm)) {
3918 rc = __kvm_ucontrol_vcpu_init(vcpu);
3919 if (rc)
3920 goto out_free_sie_block;
3921 }
3922
3923 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p",
3924 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3925 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3926
3927 rc = kvm_s390_vcpu_setup(vcpu);
3928 if (rc)
3929 goto out_ucontrol_uninit;
3930
3931 kvm_s390_update_topology_change_report(vcpu->kvm, 1);
3932 return 0;
3933
3934 out_ucontrol_uninit:
3935 if (kvm_is_ucontrol(vcpu->kvm))
3936 gmap_remove(vcpu->arch.gmap);
3937 out_free_sie_block:
3938 free_page((unsigned long)(vcpu->arch.sie_block));
3939 return rc;
3940 }
3941
kvm_arch_vcpu_runnable(struct kvm_vcpu * vcpu)3942 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3943 {
3944 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
3945 return kvm_s390_vcpu_has_irq(vcpu, 0);
3946 }
3947
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)3948 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3949 {
3950 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3951 }
3952
kvm_s390_vcpu_block(struct kvm_vcpu * vcpu)3953 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3954 {
3955 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3956 exit_sie(vcpu);
3957 }
3958
kvm_s390_vcpu_unblock(struct kvm_vcpu * vcpu)3959 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3960 {
3961 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3962 }
3963
kvm_s390_vcpu_request(struct kvm_vcpu * vcpu)3964 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3965 {
3966 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3967 exit_sie(vcpu);
3968 }
3969
kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu * vcpu)3970 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3971 {
3972 return atomic_read(&vcpu->arch.sie_block->prog20) &
3973 (PROG_BLOCK_SIE | PROG_REQUEST);
3974 }
3975
kvm_s390_vcpu_request_handled(struct kvm_vcpu * vcpu)3976 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3977 {
3978 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3979 }
3980
3981 /*
3982 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3983 * If the CPU is not running (e.g. waiting as idle) the function will
3984 * return immediately. */
exit_sie(struct kvm_vcpu * vcpu)3985 void exit_sie(struct kvm_vcpu *vcpu)
3986 {
3987 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3988 kvm_s390_vsie_kick(vcpu);
3989 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3990 cpu_relax();
3991 }
3992
3993 /* Kick a guest cpu out of SIE to process a request synchronously */
kvm_s390_sync_request(int req,struct kvm_vcpu * vcpu)3994 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3995 {
3996 __kvm_make_request(req, vcpu);
3997 kvm_s390_vcpu_request(vcpu);
3998 }
3999
kvm_gmap_notifier(struct gmap * gmap,unsigned long start,unsigned long end)4000 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
4001 unsigned long end)
4002 {
4003 struct kvm *kvm = gmap->private;
4004 struct kvm_vcpu *vcpu;
4005 unsigned long prefix;
4006 unsigned long i;
4007
4008 trace_kvm_s390_gmap_notifier(start, end, gmap_is_shadow(gmap));
4009
4010 if (gmap_is_shadow(gmap))
4011 return;
4012 if (start >= 1UL << 31)
4013 /* We are only interested in prefix pages */
4014 return;
4015 kvm_for_each_vcpu(i, vcpu, kvm) {
4016 /* match against both prefix pages */
4017 prefix = kvm_s390_get_prefix(vcpu);
4018 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
4019 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
4020 start, end);
4021 kvm_s390_sync_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
4022 }
4023 }
4024 }
4025
kvm_arch_no_poll(struct kvm_vcpu * vcpu)4026 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
4027 {
4028 /* do not poll with more than halt_poll_max_steal percent of steal time */
4029 if (get_lowcore()->avg_steal_timer * 100 / (TICK_USEC << 12) >=
4030 READ_ONCE(halt_poll_max_steal)) {
4031 vcpu->stat.halt_no_poll_steal++;
4032 return true;
4033 }
4034 return false;
4035 }
4036
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)4037 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
4038 {
4039 /* kvm common code refers to this, but never calls it */
4040 BUG();
4041 return 0;
4042 }
4043
kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)4044 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
4045 struct kvm_one_reg *reg)
4046 {
4047 int r = -EINVAL;
4048
4049 switch (reg->id) {
4050 case KVM_REG_S390_TODPR:
4051 r = put_user(vcpu->arch.sie_block->todpr,
4052 (u32 __user *)reg->addr);
4053 break;
4054 case KVM_REG_S390_EPOCHDIFF:
4055 r = put_user(vcpu->arch.sie_block->epoch,
4056 (u64 __user *)reg->addr);
4057 break;
4058 case KVM_REG_S390_CPU_TIMER:
4059 r = put_user(kvm_s390_get_cpu_timer(vcpu),
4060 (u64 __user *)reg->addr);
4061 break;
4062 case KVM_REG_S390_CLOCK_COMP:
4063 r = put_user(vcpu->arch.sie_block->ckc,
4064 (u64 __user *)reg->addr);
4065 break;
4066 case KVM_REG_S390_PFTOKEN:
4067 r = put_user(vcpu->arch.pfault_token,
4068 (u64 __user *)reg->addr);
4069 break;
4070 case KVM_REG_S390_PFCOMPARE:
4071 r = put_user(vcpu->arch.pfault_compare,
4072 (u64 __user *)reg->addr);
4073 break;
4074 case KVM_REG_S390_PFSELECT:
4075 r = put_user(vcpu->arch.pfault_select,
4076 (u64 __user *)reg->addr);
4077 break;
4078 case KVM_REG_S390_PP:
4079 r = put_user(vcpu->arch.sie_block->pp,
4080 (u64 __user *)reg->addr);
4081 break;
4082 case KVM_REG_S390_GBEA:
4083 r = put_user(vcpu->arch.sie_block->gbea,
4084 (u64 __user *)reg->addr);
4085 break;
4086 default:
4087 break;
4088 }
4089
4090 return r;
4091 }
4092
kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)4093 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
4094 struct kvm_one_reg *reg)
4095 {
4096 int r = -EINVAL;
4097 __u64 val;
4098
4099 switch (reg->id) {
4100 case KVM_REG_S390_TODPR:
4101 r = get_user(vcpu->arch.sie_block->todpr,
4102 (u32 __user *)reg->addr);
4103 break;
4104 case KVM_REG_S390_EPOCHDIFF:
4105 r = get_user(vcpu->arch.sie_block->epoch,
4106 (u64 __user *)reg->addr);
4107 break;
4108 case KVM_REG_S390_CPU_TIMER:
4109 r = get_user(val, (u64 __user *)reg->addr);
4110 if (!r)
4111 kvm_s390_set_cpu_timer(vcpu, val);
4112 break;
4113 case KVM_REG_S390_CLOCK_COMP:
4114 r = get_user(vcpu->arch.sie_block->ckc,
4115 (u64 __user *)reg->addr);
4116 break;
4117 case KVM_REG_S390_PFTOKEN:
4118 r = get_user(vcpu->arch.pfault_token,
4119 (u64 __user *)reg->addr);
4120 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4121 kvm_clear_async_pf_completion_queue(vcpu);
4122 break;
4123 case KVM_REG_S390_PFCOMPARE:
4124 r = get_user(vcpu->arch.pfault_compare,
4125 (u64 __user *)reg->addr);
4126 break;
4127 case KVM_REG_S390_PFSELECT:
4128 r = get_user(vcpu->arch.pfault_select,
4129 (u64 __user *)reg->addr);
4130 break;
4131 case KVM_REG_S390_PP:
4132 r = get_user(vcpu->arch.sie_block->pp,
4133 (u64 __user *)reg->addr);
4134 break;
4135 case KVM_REG_S390_GBEA:
4136 r = get_user(vcpu->arch.sie_block->gbea,
4137 (u64 __user *)reg->addr);
4138 break;
4139 default:
4140 break;
4141 }
4142
4143 return r;
4144 }
4145
kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu * vcpu)4146 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
4147 {
4148 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
4149 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
4150 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
4151
4152 kvm_clear_async_pf_completion_queue(vcpu);
4153 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
4154 kvm_s390_vcpu_stop(vcpu);
4155 kvm_s390_clear_local_irqs(vcpu);
4156 }
4157
kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu * vcpu)4158 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
4159 {
4160 /* Initial reset is a superset of the normal reset */
4161 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4162
4163 /*
4164 * This equals initial cpu reset in pop, but we don't switch to ESA.
4165 * We do not only reset the internal data, but also ...
4166 */
4167 vcpu->arch.sie_block->gpsw.mask = 0;
4168 vcpu->arch.sie_block->gpsw.addr = 0;
4169 kvm_s390_set_prefix(vcpu, 0);
4170 kvm_s390_set_cpu_timer(vcpu, 0);
4171 vcpu->arch.sie_block->ckc = 0;
4172 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
4173 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
4174 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
4175
4176 /* ... the data in sync regs */
4177 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
4178 vcpu->run->s.regs.ckc = 0;
4179 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
4180 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
4181 vcpu->run->psw_addr = 0;
4182 vcpu->run->psw_mask = 0;
4183 vcpu->run->s.regs.todpr = 0;
4184 vcpu->run->s.regs.cputm = 0;
4185 vcpu->run->s.regs.ckc = 0;
4186 vcpu->run->s.regs.pp = 0;
4187 vcpu->run->s.regs.gbea = 1;
4188 vcpu->run->s.regs.fpc = 0;
4189 /*
4190 * Do not reset these registers in the protected case, as some of
4191 * them are overlaid and they are not accessible in this case
4192 * anyway.
4193 */
4194 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4195 vcpu->arch.sie_block->gbea = 1;
4196 vcpu->arch.sie_block->pp = 0;
4197 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4198 vcpu->arch.sie_block->todpr = 0;
4199 }
4200 }
4201
kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu * vcpu)4202 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
4203 {
4204 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
4205
4206 /* Clear reset is a superset of the initial reset */
4207 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4208
4209 memset(®s->gprs, 0, sizeof(regs->gprs));
4210 memset(®s->vrs, 0, sizeof(regs->vrs));
4211 memset(®s->acrs, 0, sizeof(regs->acrs));
4212 memset(®s->gscb, 0, sizeof(regs->gscb));
4213
4214 regs->etoken = 0;
4215 regs->etoken_extension = 0;
4216 }
4217
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)4218 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4219 {
4220 vcpu_load(vcpu);
4221 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
4222 vcpu_put(vcpu);
4223 return 0;
4224 }
4225
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)4226 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4227 {
4228 vcpu_load(vcpu);
4229 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
4230 vcpu_put(vcpu);
4231 return 0;
4232 }
4233
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)4234 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
4235 struct kvm_sregs *sregs)
4236 {
4237 vcpu_load(vcpu);
4238
4239 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
4240 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
4241
4242 vcpu_put(vcpu);
4243 return 0;
4244 }
4245
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)4246 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
4247 struct kvm_sregs *sregs)
4248 {
4249 vcpu_load(vcpu);
4250
4251 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
4252 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
4253
4254 vcpu_put(vcpu);
4255 return 0;
4256 }
4257
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)4258 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4259 {
4260 vcpu_load(vcpu);
4261
4262 vcpu->run->s.regs.fpc = fpu->fpc;
4263 if (cpu_has_vx())
4264 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
4265 (freg_t *) fpu->fprs);
4266 else
4267 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4268
4269 vcpu_put(vcpu);
4270 return 0;
4271 }
4272
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)4273 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4274 {
4275 vcpu_load(vcpu);
4276
4277 if (cpu_has_vx())
4278 convert_vx_to_fp((freg_t *) fpu->fprs,
4279 (__vector128 *) vcpu->run->s.regs.vrs);
4280 else
4281 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
4282 fpu->fpc = vcpu->run->s.regs.fpc;
4283
4284 vcpu_put(vcpu);
4285 return 0;
4286 }
4287
kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu * vcpu,psw_t psw)4288 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
4289 {
4290 int rc = 0;
4291
4292 if (!is_vcpu_stopped(vcpu))
4293 rc = -EBUSY;
4294 else {
4295 vcpu->run->psw_mask = psw.mask;
4296 vcpu->run->psw_addr = psw.addr;
4297 }
4298 return rc;
4299 }
4300
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)4301 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
4302 struct kvm_translation *tr)
4303 {
4304 return -EINVAL; /* not implemented yet */
4305 }
4306
4307 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
4308 KVM_GUESTDBG_USE_HW_BP | \
4309 KVM_GUESTDBG_ENABLE)
4310
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)4311 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
4312 struct kvm_guest_debug *dbg)
4313 {
4314 int rc = 0;
4315
4316 vcpu_load(vcpu);
4317
4318 vcpu->guest_debug = 0;
4319 kvm_s390_clear_bp_data(vcpu);
4320
4321 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
4322 rc = -EINVAL;
4323 goto out;
4324 }
4325 if (!sclp.has_gpere) {
4326 rc = -EINVAL;
4327 goto out;
4328 }
4329
4330 if (dbg->control & KVM_GUESTDBG_ENABLE) {
4331 vcpu->guest_debug = dbg->control;
4332 /* enforce guest PER */
4333 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
4334
4335 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
4336 rc = kvm_s390_import_bp_data(vcpu, dbg);
4337 } else {
4338 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4339 vcpu->arch.guestdbg.last_bp = 0;
4340 }
4341
4342 if (rc) {
4343 vcpu->guest_debug = 0;
4344 kvm_s390_clear_bp_data(vcpu);
4345 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4346 }
4347
4348 out:
4349 vcpu_put(vcpu);
4350 return rc;
4351 }
4352
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)4353 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
4354 struct kvm_mp_state *mp_state)
4355 {
4356 int ret;
4357
4358 vcpu_load(vcpu);
4359
4360 /* CHECK_STOP and LOAD are not supported yet */
4361 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
4362 KVM_MP_STATE_OPERATING;
4363
4364 vcpu_put(vcpu);
4365 return ret;
4366 }
4367
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)4368 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
4369 struct kvm_mp_state *mp_state)
4370 {
4371 int rc = 0;
4372
4373 vcpu_load(vcpu);
4374
4375 /* user space knows about this interface - let it control the state */
4376 kvm_s390_set_user_cpu_state_ctrl(vcpu->kvm);
4377
4378 switch (mp_state->mp_state) {
4379 case KVM_MP_STATE_STOPPED:
4380 rc = kvm_s390_vcpu_stop(vcpu);
4381 break;
4382 case KVM_MP_STATE_OPERATING:
4383 rc = kvm_s390_vcpu_start(vcpu);
4384 break;
4385 case KVM_MP_STATE_LOAD:
4386 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4387 rc = -ENXIO;
4388 break;
4389 }
4390 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
4391 break;
4392 case KVM_MP_STATE_CHECK_STOP:
4393 fallthrough; /* CHECK_STOP and LOAD are not supported yet */
4394 default:
4395 rc = -ENXIO;
4396 }
4397
4398 vcpu_put(vcpu);
4399 return rc;
4400 }
4401
ibs_enabled(struct kvm_vcpu * vcpu)4402 static bool ibs_enabled(struct kvm_vcpu *vcpu)
4403 {
4404 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
4405 }
4406
__kvm_s390_fixup_fault_sync(struct gmap * gmap,gpa_t gaddr,unsigned int flags)4407 static int __kvm_s390_fixup_fault_sync(struct gmap *gmap, gpa_t gaddr, unsigned int flags)
4408 {
4409 struct kvm *kvm = gmap->private;
4410 gfn_t gfn = gpa_to_gfn(gaddr);
4411 bool unlocked;
4412 hva_t vmaddr;
4413 gpa_t tmp;
4414 int rc;
4415
4416 if (kvm_is_ucontrol(kvm)) {
4417 tmp = __gmap_translate(gmap, gaddr);
4418 gfn = gpa_to_gfn(tmp);
4419 }
4420
4421 vmaddr = gfn_to_hva(kvm, gfn);
4422 rc = fixup_user_fault(gmap->mm, vmaddr, FAULT_FLAG_WRITE, &unlocked);
4423 if (!rc)
4424 rc = __gmap_link(gmap, gaddr, vmaddr);
4425 return rc;
4426 }
4427
4428 /**
4429 * __kvm_s390_mprotect_many() - Apply specified protection to guest pages
4430 * @gmap: the gmap of the guest
4431 * @gpa: the starting guest address
4432 * @npages: how many pages to protect
4433 * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
4434 * @bits: pgste notification bits to set
4435 *
4436 * Returns: 0 in case of success, < 0 in case of error - see gmap_protect_one()
4437 *
4438 * Context: kvm->srcu and gmap->mm need to be held in read mode
4439 */
__kvm_s390_mprotect_many(struct gmap * gmap,gpa_t gpa,u8 npages,unsigned int prot,unsigned long bits)4440 int __kvm_s390_mprotect_many(struct gmap *gmap, gpa_t gpa, u8 npages, unsigned int prot,
4441 unsigned long bits)
4442 {
4443 unsigned int fault_flag = (prot & PROT_WRITE) ? FAULT_FLAG_WRITE : 0;
4444 gpa_t end = gpa + npages * PAGE_SIZE;
4445 int rc;
4446
4447 for (; gpa < end; gpa = ALIGN(gpa + 1, rc)) {
4448 rc = gmap_protect_one(gmap, gpa, prot, bits);
4449 if (rc == -EAGAIN) {
4450 __kvm_s390_fixup_fault_sync(gmap, gpa, fault_flag);
4451 rc = gmap_protect_one(gmap, gpa, prot, bits);
4452 }
4453 if (rc < 0)
4454 return rc;
4455 }
4456
4457 return 0;
4458 }
4459
kvm_s390_mprotect_notify_prefix(struct kvm_vcpu * vcpu)4460 static int kvm_s390_mprotect_notify_prefix(struct kvm_vcpu *vcpu)
4461 {
4462 gpa_t gaddr = kvm_s390_get_prefix(vcpu);
4463 int idx, rc;
4464
4465 idx = srcu_read_lock(&vcpu->kvm->srcu);
4466 mmap_read_lock(vcpu->arch.gmap->mm);
4467
4468 rc = __kvm_s390_mprotect_many(vcpu->arch.gmap, gaddr, 2, PROT_WRITE, GMAP_NOTIFY_MPROT);
4469
4470 mmap_read_unlock(vcpu->arch.gmap->mm);
4471 srcu_read_unlock(&vcpu->kvm->srcu, idx);
4472
4473 return rc;
4474 }
4475
kvm_s390_handle_requests(struct kvm_vcpu * vcpu)4476 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
4477 {
4478 retry:
4479 kvm_s390_vcpu_request_handled(vcpu);
4480 if (!kvm_request_pending(vcpu))
4481 return 0;
4482 /*
4483 * If the guest prefix changed, re-arm the ipte notifier for the
4484 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
4485 * This ensures that the ipte instruction for this request has
4486 * already finished. We might race against a second unmapper that
4487 * wants to set the blocking bit. Lets just retry the request loop.
4488 */
4489 if (kvm_check_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu)) {
4490 int rc;
4491
4492 rc = kvm_s390_mprotect_notify_prefix(vcpu);
4493 if (rc) {
4494 kvm_make_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
4495 return rc;
4496 }
4497 goto retry;
4498 }
4499
4500 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
4501 vcpu->arch.sie_block->ihcpu = 0xffff;
4502 goto retry;
4503 }
4504
4505 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
4506 if (!ibs_enabled(vcpu)) {
4507 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
4508 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
4509 }
4510 goto retry;
4511 }
4512
4513 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
4514 if (ibs_enabled(vcpu)) {
4515 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
4516 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
4517 }
4518 goto retry;
4519 }
4520
4521 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
4522 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
4523 goto retry;
4524 }
4525
4526 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
4527 /*
4528 * Disable CMM virtualization; we will emulate the ESSA
4529 * instruction manually, in order to provide additional
4530 * functionalities needed for live migration.
4531 */
4532 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
4533 goto retry;
4534 }
4535
4536 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
4537 /*
4538 * Re-enable CMM virtualization if CMMA is available and
4539 * CMM has been used.
4540 */
4541 if ((vcpu->kvm->arch.use_cmma) &&
4542 (vcpu->kvm->mm->context.uses_cmm))
4543 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
4544 goto retry;
4545 }
4546
4547 /* we left the vsie handler, nothing to do, just clear the request */
4548 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
4549
4550 return 0;
4551 }
4552
__kvm_s390_set_tod_clock(struct kvm * kvm,const struct kvm_s390_vm_tod_clock * gtod)4553 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4554 {
4555 struct kvm_vcpu *vcpu;
4556 union tod_clock clk;
4557 unsigned long i;
4558
4559 preempt_disable();
4560
4561 store_tod_clock_ext(&clk);
4562
4563 kvm->arch.epoch = gtod->tod - clk.tod;
4564 kvm->arch.epdx = 0;
4565 if (test_kvm_facility(kvm, 139)) {
4566 kvm->arch.epdx = gtod->epoch_idx - clk.ei;
4567 if (kvm->arch.epoch > gtod->tod)
4568 kvm->arch.epdx -= 1;
4569 }
4570
4571 kvm_s390_vcpu_block_all(kvm);
4572 kvm_for_each_vcpu(i, vcpu, kvm) {
4573 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
4574 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
4575 }
4576
4577 kvm_s390_vcpu_unblock_all(kvm);
4578 preempt_enable();
4579 }
4580
kvm_s390_try_set_tod_clock(struct kvm * kvm,const struct kvm_s390_vm_tod_clock * gtod)4581 int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4582 {
4583 if (!mutex_trylock(&kvm->lock))
4584 return 0;
4585 __kvm_s390_set_tod_clock(kvm, gtod);
4586 mutex_unlock(&kvm->lock);
4587 return 1;
4588 }
4589
__kvm_inject_pfault_token(struct kvm_vcpu * vcpu,bool start_token,unsigned long token)4590 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
4591 unsigned long token)
4592 {
4593 struct kvm_s390_interrupt inti;
4594 struct kvm_s390_irq irq;
4595
4596 if (start_token) {
4597 irq.u.ext.ext_params2 = token;
4598 irq.type = KVM_S390_INT_PFAULT_INIT;
4599 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
4600 } else {
4601 inti.type = KVM_S390_INT_PFAULT_DONE;
4602 inti.parm64 = token;
4603 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
4604 }
4605 }
4606
kvm_arch_async_page_not_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4607 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
4608 struct kvm_async_pf *work)
4609 {
4610 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
4611 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
4612
4613 return true;
4614 }
4615
kvm_arch_async_page_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4616 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
4617 struct kvm_async_pf *work)
4618 {
4619 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
4620 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
4621 }
4622
kvm_arch_async_page_ready(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4623 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
4624 struct kvm_async_pf *work)
4625 {
4626 /* s390 will always inject the page directly */
4627 }
4628
kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu * vcpu)4629 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
4630 {
4631 /*
4632 * s390 will always inject the page directly,
4633 * but we still want check_async_completion to cleanup
4634 */
4635 return true;
4636 }
4637
kvm_arch_setup_async_pf(struct kvm_vcpu * vcpu)4638 static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
4639 {
4640 hva_t hva;
4641 struct kvm_arch_async_pf arch;
4642
4643 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4644 return false;
4645 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
4646 vcpu->arch.pfault_compare)
4647 return false;
4648 if (psw_extint_disabled(vcpu))
4649 return false;
4650 if (kvm_s390_vcpu_has_irq(vcpu, 0))
4651 return false;
4652 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
4653 return false;
4654 if (!vcpu->arch.gmap->pfault_enabled)
4655 return false;
4656
4657 hva = gfn_to_hva(vcpu->kvm, current->thread.gmap_teid.addr);
4658 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
4659 return false;
4660
4661 return kvm_setup_async_pf(vcpu, current->thread.gmap_teid.addr * PAGE_SIZE, hva, &arch);
4662 }
4663
vcpu_pre_run(struct kvm_vcpu * vcpu)4664 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
4665 {
4666 int rc, cpuflags;
4667
4668 /*
4669 * On s390 notifications for arriving pages will be delivered directly
4670 * to the guest but the house keeping for completed pfaults is
4671 * handled outside the worker.
4672 */
4673 kvm_check_async_pf_completion(vcpu);
4674
4675 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
4676 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
4677
4678 if (!kvm_is_ucontrol(vcpu->kvm)) {
4679 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4680 if (rc || guestdbg_exit_pending(vcpu))
4681 return rc;
4682 }
4683
4684 rc = kvm_s390_handle_requests(vcpu);
4685 if (rc)
4686 return rc;
4687
4688 if (guestdbg_enabled(vcpu)) {
4689 kvm_s390_backup_guest_per_regs(vcpu);
4690 kvm_s390_patch_guest_per_regs(vcpu);
4691 }
4692
4693 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
4694
4695 vcpu->arch.sie_block->icptcode = 0;
4696 current->thread.gmap_int_code = 0;
4697 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4698 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4699 trace_kvm_s390_sie_enter(vcpu, cpuflags);
4700
4701 return 0;
4702 }
4703
vcpu_post_run_addressing_exception(struct kvm_vcpu * vcpu)4704 static int vcpu_post_run_addressing_exception(struct kvm_vcpu *vcpu)
4705 {
4706 struct kvm_s390_pgm_info pgm_info = {
4707 .code = PGM_ADDRESSING,
4708 };
4709 u8 opcode, ilen;
4710 int rc;
4711
4712 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4713 trace_kvm_s390_sie_fault(vcpu);
4714
4715 /*
4716 * We want to inject an addressing exception, which is defined as a
4717 * suppressing or terminating exception. However, since we came here
4718 * by a DAT access exception, the PSW still points to the faulting
4719 * instruction since DAT exceptions are nullifying. So we've got
4720 * to look up the current opcode to get the length of the instruction
4721 * to be able to forward the PSW.
4722 */
4723 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
4724 ilen = insn_length(opcode);
4725 if (rc < 0) {
4726 return rc;
4727 } else if (rc) {
4728 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4729 * Forward by arbitrary ilc, injection will take care of
4730 * nullification if necessary.
4731 */
4732 pgm_info = vcpu->arch.pgm;
4733 ilen = 4;
4734 }
4735 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4736 kvm_s390_forward_psw(vcpu, ilen);
4737 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
4738 }
4739
kvm_s390_assert_primary_as(struct kvm_vcpu * vcpu)4740 static void kvm_s390_assert_primary_as(struct kvm_vcpu *vcpu)
4741 {
4742 KVM_BUG(current->thread.gmap_teid.as != PSW_BITS_AS_PRIMARY, vcpu->kvm,
4743 "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4744 current->thread.gmap_int_code, current->thread.gmap_teid.val);
4745 }
4746
4747 /*
4748 * __kvm_s390_handle_dat_fault() - handle a dat fault for the gmap of a vcpu
4749 * @vcpu: the vCPU whose gmap is to be fixed up
4750 * @gfn: the guest frame number used for memslots (including fake memslots)
4751 * @gaddr: the gmap address, does not have to match @gfn for ucontrol gmaps
4752 * @foll: FOLL_* flags
4753 *
4754 * Return: 0 on success, < 0 in case of error.
4755 * Context: The mm lock must not be held before calling. May sleep.
4756 */
__kvm_s390_handle_dat_fault(struct kvm_vcpu * vcpu,gfn_t gfn,gpa_t gaddr,unsigned int foll)4757 int __kvm_s390_handle_dat_fault(struct kvm_vcpu *vcpu, gfn_t gfn, gpa_t gaddr, unsigned int foll)
4758 {
4759 struct kvm_memory_slot *slot;
4760 unsigned int fault_flags;
4761 bool writable, unlocked;
4762 unsigned long vmaddr;
4763 struct page *page;
4764 kvm_pfn_t pfn;
4765 int rc;
4766
4767 slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn);
4768 if (!slot || slot->flags & KVM_MEMSLOT_INVALID)
4769 return vcpu_post_run_addressing_exception(vcpu);
4770
4771 fault_flags = foll & FOLL_WRITE ? FAULT_FLAG_WRITE : 0;
4772 if (vcpu->arch.gmap->pfault_enabled)
4773 foll |= FOLL_NOWAIT;
4774 vmaddr = __gfn_to_hva_memslot(slot, gfn);
4775
4776 try_again:
4777 pfn = __kvm_faultin_pfn(slot, gfn, foll, &writable, &page);
4778
4779 /* Access outside memory, inject addressing exception */
4780 if (is_noslot_pfn(pfn))
4781 return vcpu_post_run_addressing_exception(vcpu);
4782 /* Signal pending: try again */
4783 if (pfn == KVM_PFN_ERR_SIGPENDING)
4784 return -EAGAIN;
4785
4786 /* Needs I/O, try to setup async pfault (only possible with FOLL_NOWAIT) */
4787 if (pfn == KVM_PFN_ERR_NEEDS_IO) {
4788 trace_kvm_s390_major_guest_pfault(vcpu);
4789 if (kvm_arch_setup_async_pf(vcpu))
4790 return 0;
4791 vcpu->stat.pfault_sync++;
4792 /* Could not setup async pfault, try again synchronously */
4793 foll &= ~FOLL_NOWAIT;
4794 goto try_again;
4795 }
4796 /* Any other error */
4797 if (is_error_pfn(pfn))
4798 return -EFAULT;
4799
4800 /* Success */
4801 mmap_read_lock(vcpu->arch.gmap->mm);
4802 /* Mark the userspace PTEs as young and/or dirty, to avoid page fault loops */
4803 rc = fixup_user_fault(vcpu->arch.gmap->mm, vmaddr, fault_flags, &unlocked);
4804 if (!rc)
4805 rc = __gmap_link(vcpu->arch.gmap, gaddr, vmaddr);
4806 scoped_guard(spinlock, &vcpu->kvm->mmu_lock) {
4807 kvm_release_faultin_page(vcpu->kvm, page, false, writable);
4808 }
4809 mmap_read_unlock(vcpu->arch.gmap->mm);
4810 return rc;
4811 }
4812
vcpu_dat_fault_handler(struct kvm_vcpu * vcpu,unsigned long gaddr,unsigned int foll)4813 static int vcpu_dat_fault_handler(struct kvm_vcpu *vcpu, unsigned long gaddr, unsigned int foll)
4814 {
4815 unsigned long gaddr_tmp;
4816 gfn_t gfn;
4817
4818 gfn = gpa_to_gfn(gaddr);
4819 if (kvm_is_ucontrol(vcpu->kvm)) {
4820 /*
4821 * This translates the per-vCPU guest address into a
4822 * fake guest address, which can then be used with the
4823 * fake memslots that are identity mapping userspace.
4824 * This allows ucontrol VMs to use the normal fault
4825 * resolution path, like normal VMs.
4826 */
4827 mmap_read_lock(vcpu->arch.gmap->mm);
4828 gaddr_tmp = __gmap_translate(vcpu->arch.gmap, gaddr);
4829 mmap_read_unlock(vcpu->arch.gmap->mm);
4830 if (gaddr_tmp == -EFAULT) {
4831 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4832 vcpu->run->s390_ucontrol.trans_exc_code = gaddr;
4833 vcpu->run->s390_ucontrol.pgm_code = PGM_SEGMENT_TRANSLATION;
4834 return -EREMOTE;
4835 }
4836 gfn = gpa_to_gfn(gaddr_tmp);
4837 }
4838 return __kvm_s390_handle_dat_fault(vcpu, gfn, gaddr, foll);
4839 }
4840
vcpu_post_run_handle_fault(struct kvm_vcpu * vcpu)4841 static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu)
4842 {
4843 unsigned int foll = 0;
4844 unsigned long gaddr;
4845 int rc;
4846
4847 gaddr = current->thread.gmap_teid.addr * PAGE_SIZE;
4848 if (kvm_s390_cur_gmap_fault_is_write())
4849 foll = FOLL_WRITE;
4850
4851 switch (current->thread.gmap_int_code & PGM_INT_CODE_MASK) {
4852 case 0:
4853 vcpu->stat.exit_null++;
4854 break;
4855 case PGM_SECURE_STORAGE_ACCESS:
4856 case PGM_SECURE_STORAGE_VIOLATION:
4857 kvm_s390_assert_primary_as(vcpu);
4858 /*
4859 * This can happen after a reboot with asynchronous teardown;
4860 * the new guest (normal or protected) will run on top of the
4861 * previous protected guest. The old pages need to be destroyed
4862 * so the new guest can use them.
4863 */
4864 if (kvm_s390_pv_destroy_page(vcpu->kvm, gaddr)) {
4865 /*
4866 * Either KVM messed up the secure guest mapping or the
4867 * same page is mapped into multiple secure guests.
4868 *
4869 * This exception is only triggered when a guest 2 is
4870 * running and can therefore never occur in kernel
4871 * context.
4872 */
4873 pr_warn_ratelimited("Secure storage violation (%x) in task: %s, pid %d\n",
4874 current->thread.gmap_int_code, current->comm,
4875 current->pid);
4876 send_sig(SIGSEGV, current, 0);
4877 }
4878 break;
4879 case PGM_NON_SECURE_STORAGE_ACCESS:
4880 kvm_s390_assert_primary_as(vcpu);
4881 /*
4882 * This is normal operation; a page belonging to a protected
4883 * guest has not been imported yet. Try to import the page into
4884 * the protected guest.
4885 */
4886 rc = kvm_s390_pv_convert_to_secure(vcpu->kvm, gaddr);
4887 if (rc == -EINVAL)
4888 send_sig(SIGSEGV, current, 0);
4889 if (rc != -ENXIO)
4890 break;
4891 foll = FOLL_WRITE;
4892 fallthrough;
4893 case PGM_PROTECTION:
4894 case PGM_SEGMENT_TRANSLATION:
4895 case PGM_PAGE_TRANSLATION:
4896 case PGM_ASCE_TYPE:
4897 case PGM_REGION_FIRST_TRANS:
4898 case PGM_REGION_SECOND_TRANS:
4899 case PGM_REGION_THIRD_TRANS:
4900 kvm_s390_assert_primary_as(vcpu);
4901 return vcpu_dat_fault_handler(vcpu, gaddr, foll);
4902 default:
4903 KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4904 current->thread.gmap_int_code, current->thread.gmap_teid.val);
4905 send_sig(SIGSEGV, current, 0);
4906 break;
4907 }
4908 return 0;
4909 }
4910
vcpu_post_run(struct kvm_vcpu * vcpu,int exit_reason)4911 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
4912 {
4913 struct mcck_volatile_info *mcck_info;
4914 struct sie_page *sie_page;
4915 int rc;
4916
4917 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4918 vcpu->arch.sie_block->icptcode);
4919 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4920
4921 if (guestdbg_enabled(vcpu))
4922 kvm_s390_restore_guest_per_regs(vcpu);
4923
4924 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4925 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
4926
4927 if (exit_reason == -EINTR) {
4928 VCPU_EVENT(vcpu, 3, "%s", "machine check");
4929 sie_page = container_of(vcpu->arch.sie_block,
4930 struct sie_page, sie_block);
4931 mcck_info = &sie_page->mcck_info;
4932 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4933 return 0;
4934 }
4935
4936 if (vcpu->arch.sie_block->icptcode > 0) {
4937 rc = kvm_handle_sie_intercept(vcpu);
4938
4939 if (rc != -EOPNOTSUPP)
4940 return rc;
4941 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4942 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4943 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4944 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4945 return -EREMOTE;
4946 }
4947
4948 return vcpu_post_run_handle_fault(vcpu);
4949 }
4950
kvm_s390_enter_exit_sie(struct kvm_s390_sie_block * scb,u64 * gprs,unsigned long gasce)4951 int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
4952 u64 *gprs, unsigned long gasce)
4953 {
4954 int ret;
4955
4956 guest_state_enter_irqoff();
4957
4958 /*
4959 * The guest_state_{enter,exit}_irqoff() functions inform lockdep and
4960 * tracing that entry to the guest will enable host IRQs, and exit from
4961 * the guest will disable host IRQs.
4962 */
4963 ret = sie64a(scb, gprs, gasce);
4964
4965 guest_state_exit_irqoff();
4966
4967 return ret;
4968 }
4969
4970 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
__vcpu_run(struct kvm_vcpu * vcpu)4971 static int __vcpu_run(struct kvm_vcpu *vcpu)
4972 {
4973 int rc, exit_reason;
4974 struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
4975
4976 /*
4977 * We try to hold kvm->srcu during most of vcpu_run (except when run-
4978 * ning the guest), so that memslots (and other stuff) are protected
4979 */
4980 kvm_vcpu_srcu_read_lock(vcpu);
4981
4982 while (true) {
4983 rc = vcpu_pre_run(vcpu);
4984 kvm_vcpu_srcu_read_unlock(vcpu);
4985 if (rc || guestdbg_exit_pending(vcpu))
4986 break;
4987
4988 /*
4989 * As PF_VCPU will be used in fault handler, between
4990 * guest_timing_enter_irqoff and guest_timing_exit_irqoff
4991 * should be no uaccess.
4992 */
4993 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4994 memcpy(sie_page->pv_grregs,
4995 vcpu->run->s.regs.gprs,
4996 sizeof(sie_page->pv_grregs));
4997 }
4998
4999 xfer_to_guest_mode_check:
5000 local_irq_disable();
5001 xfer_to_guest_mode_prepare();
5002 if (xfer_to_guest_mode_work_pending()) {
5003 local_irq_enable();
5004 rc = kvm_xfer_to_guest_mode_handle_work(vcpu);
5005 if (rc)
5006 break;
5007 goto xfer_to_guest_mode_check;
5008 }
5009
5010 guest_timing_enter_irqoff();
5011 __disable_cpu_timer_accounting(vcpu);
5012
5013 exit_reason = kvm_s390_enter_exit_sie(vcpu->arch.sie_block,
5014 vcpu->run->s.regs.gprs,
5015 vcpu->arch.gmap->asce);
5016
5017 __enable_cpu_timer_accounting(vcpu);
5018 guest_timing_exit_irqoff();
5019 local_irq_enable();
5020
5021 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5022 memcpy(vcpu->run->s.regs.gprs,
5023 sie_page->pv_grregs,
5024 sizeof(sie_page->pv_grregs));
5025 /*
5026 * We're not allowed to inject interrupts on intercepts
5027 * that leave the guest state in an "in-between" state
5028 * where the next SIE entry will do a continuation.
5029 * Fence interrupts in our "internal" PSW.
5030 */
5031 if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
5032 vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
5033 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
5034 }
5035 }
5036 kvm_vcpu_srcu_read_lock(vcpu);
5037
5038 rc = vcpu_post_run(vcpu, exit_reason);
5039 if (rc || guestdbg_exit_pending(vcpu)) {
5040 kvm_vcpu_srcu_read_unlock(vcpu);
5041 break;
5042 }
5043 }
5044
5045 return rc;
5046 }
5047
sync_regs_fmt2(struct kvm_vcpu * vcpu)5048 static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
5049 {
5050 struct kvm_run *kvm_run = vcpu->run;
5051 struct runtime_instr_cb *riccb;
5052 struct gs_cb *gscb;
5053
5054 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
5055 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
5056 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
5057 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
5058 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
5059 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
5060 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
5061 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
5062 }
5063 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
5064 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
5065 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
5066 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
5067 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
5068 kvm_clear_async_pf_completion_queue(vcpu);
5069 }
5070 if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
5071 vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
5072 vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
5073 VCPU_EVENT(vcpu, 3, "setting cpnc to %d", vcpu->arch.diag318_info.cpnc);
5074 }
5075 /*
5076 * If userspace sets the riccb (e.g. after migration) to a valid state,
5077 * we should enable RI here instead of doing the lazy enablement.
5078 */
5079 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
5080 test_kvm_facility(vcpu->kvm, 64) &&
5081 riccb->v &&
5082 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
5083 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
5084 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
5085 }
5086 /*
5087 * If userspace sets the gscb (e.g. after migration) to non-zero,
5088 * we should enable GS here instead of doing the lazy enablement.
5089 */
5090 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
5091 test_kvm_facility(vcpu->kvm, 133) &&
5092 gscb->gssm &&
5093 !vcpu->arch.gs_enabled) {
5094 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
5095 vcpu->arch.sie_block->ecb |= ECB_GS;
5096 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
5097 vcpu->arch.gs_enabled = 1;
5098 }
5099 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
5100 test_kvm_facility(vcpu->kvm, 82)) {
5101 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
5102 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
5103 }
5104 if (cpu_has_gs()) {
5105 preempt_disable();
5106 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
5107 if (current->thread.gs_cb) {
5108 vcpu->arch.host_gscb = current->thread.gs_cb;
5109 save_gs_cb(vcpu->arch.host_gscb);
5110 }
5111 if (vcpu->arch.gs_enabled) {
5112 current->thread.gs_cb = (struct gs_cb *)
5113 &vcpu->run->s.regs.gscb;
5114 restore_gs_cb(current->thread.gs_cb);
5115 }
5116 preempt_enable();
5117 }
5118 /* SIE will load etoken directly from SDNX and therefore kvm_run */
5119 }
5120
sync_regs(struct kvm_vcpu * vcpu)5121 static void sync_regs(struct kvm_vcpu *vcpu)
5122 {
5123 struct kvm_run *kvm_run = vcpu->run;
5124
5125 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
5126 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
5127 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
5128 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
5129 /* some control register changes require a tlb flush */
5130 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
5131 }
5132 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
5133 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
5134 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
5135 }
5136 save_access_regs(vcpu->arch.host_acrs);
5137 restore_access_regs(vcpu->run->s.regs.acrs);
5138 vcpu->arch.acrs_loaded = true;
5139 kvm_s390_fpu_load(vcpu->run);
5140 /* Sync fmt2 only data */
5141 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
5142 sync_regs_fmt2(vcpu);
5143 } else {
5144 /*
5145 * In several places we have to modify our internal view to
5146 * not do things that are disallowed by the ultravisor. For
5147 * example we must not inject interrupts after specific exits
5148 * (e.g. 112 prefix page not secure). We do this by turning
5149 * off the machine check, external and I/O interrupt bits
5150 * of our PSW copy. To avoid getting validity intercepts, we
5151 * do only accept the condition code from userspace.
5152 */
5153 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
5154 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
5155 PSW_MASK_CC;
5156 }
5157
5158 kvm_run->kvm_dirty_regs = 0;
5159 }
5160
store_regs_fmt2(struct kvm_vcpu * vcpu)5161 static void store_regs_fmt2(struct kvm_vcpu *vcpu)
5162 {
5163 struct kvm_run *kvm_run = vcpu->run;
5164
5165 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
5166 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
5167 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
5168 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
5169 kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
5170 if (cpu_has_gs()) {
5171 preempt_disable();
5172 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
5173 if (vcpu->arch.gs_enabled)
5174 save_gs_cb(current->thread.gs_cb);
5175 current->thread.gs_cb = vcpu->arch.host_gscb;
5176 restore_gs_cb(vcpu->arch.host_gscb);
5177 if (!vcpu->arch.host_gscb)
5178 local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT);
5179 vcpu->arch.host_gscb = NULL;
5180 preempt_enable();
5181 }
5182 /* SIE will save etoken directly into SDNX and therefore kvm_run */
5183 }
5184
store_regs(struct kvm_vcpu * vcpu)5185 static void store_regs(struct kvm_vcpu *vcpu)
5186 {
5187 struct kvm_run *kvm_run = vcpu->run;
5188
5189 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
5190 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
5191 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
5192 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
5193 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
5194 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
5195 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
5196 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
5197 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
5198 save_access_regs(vcpu->run->s.regs.acrs);
5199 restore_access_regs(vcpu->arch.host_acrs);
5200 vcpu->arch.acrs_loaded = false;
5201 kvm_s390_fpu_store(vcpu->run);
5202 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
5203 store_regs_fmt2(vcpu);
5204 }
5205
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)5206 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
5207 {
5208 struct kvm_run *kvm_run = vcpu->run;
5209 DECLARE_KERNEL_FPU_ONSTACK32(fpu);
5210 int rc;
5211
5212 /*
5213 * Running a VM while dumping always has the potential to
5214 * produce inconsistent dump data. But for PV vcpus a SIE
5215 * entry while dumping could also lead to a fatal validity
5216 * intercept which we absolutely want to avoid.
5217 */
5218 if (vcpu->kvm->arch.pv.dumping)
5219 return -EINVAL;
5220
5221 if (!vcpu->wants_to_run)
5222 return -EINTR;
5223
5224 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
5225 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
5226 return -EINVAL;
5227
5228 vcpu_load(vcpu);
5229
5230 if (guestdbg_exit_pending(vcpu)) {
5231 kvm_s390_prepare_debug_exit(vcpu);
5232 rc = 0;
5233 goto out;
5234 }
5235
5236 kvm_sigset_activate(vcpu);
5237
5238 /*
5239 * no need to check the return value of vcpu_start as it can only have
5240 * an error for protvirt, but protvirt means user cpu state
5241 */
5242 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
5243 kvm_s390_vcpu_start(vcpu);
5244 } else if (is_vcpu_stopped(vcpu)) {
5245 pr_err_ratelimited("can't run stopped vcpu %d\n",
5246 vcpu->vcpu_id);
5247 rc = -EINVAL;
5248 goto out;
5249 }
5250
5251 kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR);
5252 sync_regs(vcpu);
5253 enable_cpu_timer_accounting(vcpu);
5254
5255 might_fault();
5256 rc = __vcpu_run(vcpu);
5257
5258 if (signal_pending(current) && !rc) {
5259 kvm_run->exit_reason = KVM_EXIT_INTR;
5260 vcpu->stat.signal_exits++;
5261 rc = -EINTR;
5262 }
5263
5264 if (guestdbg_exit_pending(vcpu) && !rc) {
5265 kvm_s390_prepare_debug_exit(vcpu);
5266 rc = 0;
5267 }
5268
5269 if (rc == -EREMOTE) {
5270 /* userspace support is needed, kvm_run has been prepared */
5271 rc = 0;
5272 }
5273
5274 disable_cpu_timer_accounting(vcpu);
5275 store_regs(vcpu);
5276 kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR);
5277
5278 kvm_sigset_deactivate(vcpu);
5279
5280 vcpu->stat.exit_userspace++;
5281 out:
5282 vcpu_put(vcpu);
5283 return rc;
5284 }
5285
5286 /*
5287 * store status at address
5288 * we use have two special cases:
5289 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
5290 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
5291 */
kvm_s390_store_status_unloaded(struct kvm_vcpu * vcpu,unsigned long gpa)5292 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
5293 {
5294 unsigned char archmode = 1;
5295 freg_t fprs[NUM_FPRS];
5296 unsigned int px;
5297 u64 clkcomp, cputm;
5298 int rc;
5299
5300 px = kvm_s390_get_prefix(vcpu);
5301 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
5302 if (write_guest_abs(vcpu, 163, &archmode, 1))
5303 return -EFAULT;
5304 gpa = 0;
5305 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
5306 if (write_guest_real(vcpu, 163, &archmode, 1))
5307 return -EFAULT;
5308 gpa = px;
5309 } else
5310 gpa -= __LC_FPREGS_SAVE_AREA;
5311
5312 /* manually convert vector registers if necessary */
5313 if (cpu_has_vx()) {
5314 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
5315 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5316 fprs, 128);
5317 } else {
5318 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5319 vcpu->run->s.regs.fprs, 128);
5320 }
5321 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
5322 vcpu->run->s.regs.gprs, 128);
5323 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
5324 &vcpu->arch.sie_block->gpsw, 16);
5325 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
5326 &px, 4);
5327 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
5328 &vcpu->run->s.regs.fpc, 4);
5329 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
5330 &vcpu->arch.sie_block->todpr, 4);
5331 cputm = kvm_s390_get_cpu_timer(vcpu);
5332 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
5333 &cputm, 8);
5334 clkcomp = vcpu->arch.sie_block->ckc >> 8;
5335 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
5336 &clkcomp, 8);
5337 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
5338 &vcpu->run->s.regs.acrs, 64);
5339 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
5340 &vcpu->arch.sie_block->gcr, 128);
5341 return rc ? -EFAULT : 0;
5342 }
5343
kvm_s390_vcpu_store_status(struct kvm_vcpu * vcpu,unsigned long addr)5344 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
5345 {
5346 /*
5347 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
5348 * switch in the run ioctl. Let's update our copies before we save
5349 * it into the save area
5350 */
5351 kvm_s390_fpu_store(vcpu->run);
5352 save_access_regs(vcpu->run->s.regs.acrs);
5353
5354 return kvm_s390_store_status_unloaded(vcpu, addr);
5355 }
5356
__disable_ibs_on_vcpu(struct kvm_vcpu * vcpu)5357 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5358 {
5359 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
5360 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
5361 }
5362
__disable_ibs_on_all_vcpus(struct kvm * kvm)5363 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
5364 {
5365 unsigned long i;
5366 struct kvm_vcpu *vcpu;
5367
5368 kvm_for_each_vcpu(i, vcpu, kvm) {
5369 __disable_ibs_on_vcpu(vcpu);
5370 }
5371 }
5372
__enable_ibs_on_vcpu(struct kvm_vcpu * vcpu)5373 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5374 {
5375 if (!sclp.has_ibs)
5376 return;
5377 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
5378 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
5379 }
5380
kvm_s390_vcpu_start(struct kvm_vcpu * vcpu)5381 int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
5382 {
5383 int i, online_vcpus, r = 0, started_vcpus = 0;
5384
5385 if (!is_vcpu_stopped(vcpu))
5386 return 0;
5387
5388 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
5389 /* Only one cpu at a time may enter/leave the STOPPED state. */
5390 spin_lock(&vcpu->kvm->arch.start_stop_lock);
5391 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
5392
5393 /* Let's tell the UV that we want to change into the operating state */
5394 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5395 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
5396 if (r) {
5397 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5398 return r;
5399 }
5400 }
5401
5402 for (i = 0; i < online_vcpus; i++) {
5403 if (!is_vcpu_stopped(kvm_get_vcpu(vcpu->kvm, i)))
5404 started_vcpus++;
5405 }
5406
5407 if (started_vcpus == 0) {
5408 /* we're the only active VCPU -> speed it up */
5409 __enable_ibs_on_vcpu(vcpu);
5410 } else if (started_vcpus == 1) {
5411 /*
5412 * As we are starting a second VCPU, we have to disable
5413 * the IBS facility on all VCPUs to remove potentially
5414 * outstanding ENABLE requests.
5415 */
5416 __disable_ibs_on_all_vcpus(vcpu->kvm);
5417 }
5418
5419 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
5420 /*
5421 * The real PSW might have changed due to a RESTART interpreted by the
5422 * ultravisor. We block all interrupts and let the next sie exit
5423 * refresh our view.
5424 */
5425 if (kvm_s390_pv_cpu_is_protected(vcpu))
5426 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
5427 /*
5428 * Another VCPU might have used IBS while we were offline.
5429 * Let's play safe and flush the VCPU at startup.
5430 */
5431 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
5432 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5433 return 0;
5434 }
5435
kvm_s390_vcpu_stop(struct kvm_vcpu * vcpu)5436 int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
5437 {
5438 int i, online_vcpus, r = 0, started_vcpus = 0;
5439 struct kvm_vcpu *started_vcpu = NULL;
5440
5441 if (is_vcpu_stopped(vcpu))
5442 return 0;
5443
5444 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
5445 /* Only one cpu at a time may enter/leave the STOPPED state. */
5446 spin_lock(&vcpu->kvm->arch.start_stop_lock);
5447 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
5448
5449 /* Let's tell the UV that we want to change into the stopped state */
5450 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5451 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
5452 if (r) {
5453 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5454 return r;
5455 }
5456 }
5457
5458 /*
5459 * Set the VCPU to STOPPED and THEN clear the interrupt flag,
5460 * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
5461 * have been fully processed. This will ensure that the VCPU
5462 * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
5463 */
5464 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
5465 kvm_s390_clear_stop_irq(vcpu);
5466
5467 __disable_ibs_on_vcpu(vcpu);
5468
5469 for (i = 0; i < online_vcpus; i++) {
5470 struct kvm_vcpu *tmp = kvm_get_vcpu(vcpu->kvm, i);
5471
5472 if (!is_vcpu_stopped(tmp)) {
5473 started_vcpus++;
5474 started_vcpu = tmp;
5475 }
5476 }
5477
5478 if (started_vcpus == 1) {
5479 /*
5480 * As we only have one VCPU left, we want to enable the
5481 * IBS facility for that VCPU to speed it up.
5482 */
5483 __enable_ibs_on_vcpu(started_vcpu);
5484 }
5485
5486 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5487 return 0;
5488 }
5489
kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu * vcpu,struct kvm_enable_cap * cap)5490 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5491 struct kvm_enable_cap *cap)
5492 {
5493 int r;
5494
5495 if (cap->flags)
5496 return -EINVAL;
5497
5498 switch (cap->cap) {
5499 case KVM_CAP_S390_CSS_SUPPORT:
5500 if (!vcpu->kvm->arch.css_support) {
5501 vcpu->kvm->arch.css_support = 1;
5502 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
5503 trace_kvm_s390_enable_css(vcpu->kvm);
5504 }
5505 r = 0;
5506 break;
5507 default:
5508 r = -EINVAL;
5509 break;
5510 }
5511 return r;
5512 }
5513
kvm_s390_vcpu_sida_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5514 static long kvm_s390_vcpu_sida_op(struct kvm_vcpu *vcpu,
5515 struct kvm_s390_mem_op *mop)
5516 {
5517 void __user *uaddr = (void __user *)mop->buf;
5518 void *sida_addr;
5519 int r = 0;
5520
5521 if (mop->flags || !mop->size)
5522 return -EINVAL;
5523 if (mop->size + mop->sida_offset < mop->size)
5524 return -EINVAL;
5525 if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
5526 return -E2BIG;
5527 if (!kvm_s390_pv_cpu_is_protected(vcpu))
5528 return -EINVAL;
5529
5530 sida_addr = (char *)sida_addr(vcpu->arch.sie_block) + mop->sida_offset;
5531
5532 switch (mop->op) {
5533 case KVM_S390_MEMOP_SIDA_READ:
5534 if (copy_to_user(uaddr, sida_addr, mop->size))
5535 r = -EFAULT;
5536
5537 break;
5538 case KVM_S390_MEMOP_SIDA_WRITE:
5539 if (copy_from_user(sida_addr, uaddr, mop->size))
5540 r = -EFAULT;
5541 break;
5542 }
5543 return r;
5544 }
5545
kvm_s390_vcpu_mem_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5546 static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu,
5547 struct kvm_s390_mem_op *mop)
5548 {
5549 void __user *uaddr = (void __user *)mop->buf;
5550 enum gacc_mode acc_mode;
5551 void *tmpbuf = NULL;
5552 int r;
5553
5554 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_INJECT_EXCEPTION |
5555 KVM_S390_MEMOP_F_CHECK_ONLY |
5556 KVM_S390_MEMOP_F_SKEY_PROTECTION);
5557 if (r)
5558 return r;
5559 if (mop->ar >= NUM_ACRS)
5560 return -EINVAL;
5561 if (kvm_s390_pv_cpu_is_protected(vcpu))
5562 return -EINVAL;
5563 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
5564 tmpbuf = vmalloc(mop->size);
5565 if (!tmpbuf)
5566 return -ENOMEM;
5567 }
5568
5569 acc_mode = mop->op == KVM_S390_MEMOP_LOGICAL_READ ? GACC_FETCH : GACC_STORE;
5570 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
5571 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size,
5572 acc_mode, mop->key);
5573 goto out_inject;
5574 }
5575 if (acc_mode == GACC_FETCH) {
5576 r = read_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
5577 mop->size, mop->key);
5578 if (r)
5579 goto out_inject;
5580 if (copy_to_user(uaddr, tmpbuf, mop->size)) {
5581 r = -EFAULT;
5582 goto out_free;
5583 }
5584 } else {
5585 if (copy_from_user(tmpbuf, uaddr, mop->size)) {
5586 r = -EFAULT;
5587 goto out_free;
5588 }
5589 r = write_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
5590 mop->size, mop->key);
5591 }
5592
5593 out_inject:
5594 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
5595 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
5596
5597 out_free:
5598 vfree(tmpbuf);
5599 return r;
5600 }
5601
kvm_s390_vcpu_memsida_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5602 static long kvm_s390_vcpu_memsida_op(struct kvm_vcpu *vcpu,
5603 struct kvm_s390_mem_op *mop)
5604 {
5605 int r, srcu_idx;
5606
5607 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
5608
5609 switch (mop->op) {
5610 case KVM_S390_MEMOP_LOGICAL_READ:
5611 case KVM_S390_MEMOP_LOGICAL_WRITE:
5612 r = kvm_s390_vcpu_mem_op(vcpu, mop);
5613 break;
5614 case KVM_S390_MEMOP_SIDA_READ:
5615 case KVM_S390_MEMOP_SIDA_WRITE:
5616 /* we are locked against sida going away by the vcpu->mutex */
5617 r = kvm_s390_vcpu_sida_op(vcpu, mop);
5618 break;
5619 default:
5620 r = -EINVAL;
5621 }
5622
5623 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
5624 return r;
5625 }
5626
kvm_arch_vcpu_unlocked_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5627 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
5628 unsigned long arg)
5629 {
5630 struct kvm_vcpu *vcpu = filp->private_data;
5631 void __user *argp = (void __user *)arg;
5632 int rc;
5633
5634 switch (ioctl) {
5635 case KVM_S390_IRQ: {
5636 struct kvm_s390_irq s390irq;
5637
5638 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
5639 return -EFAULT;
5640 rc = kvm_s390_inject_vcpu(vcpu, &s390irq);
5641 break;
5642 }
5643 case KVM_S390_INTERRUPT: {
5644 struct kvm_s390_interrupt s390int;
5645 struct kvm_s390_irq s390irq = {};
5646
5647 if (copy_from_user(&s390int, argp, sizeof(s390int)))
5648 return -EFAULT;
5649 if (s390int_to_s390irq(&s390int, &s390irq))
5650 return -EINVAL;
5651 rc = kvm_s390_inject_vcpu(vcpu, &s390irq);
5652 break;
5653 }
5654 default:
5655 rc = -ENOIOCTLCMD;
5656 break;
5657 }
5658
5659 /*
5660 * To simplify single stepping of userspace-emulated instructions,
5661 * KVM_EXIT_S390_SIEIC exit sets KVM_GUESTDBG_EXIT_PENDING (see
5662 * should_handle_per_ifetch()). However, if userspace emulation injects
5663 * an interrupt, it needs to be cleared, so that KVM_EXIT_DEBUG happens
5664 * after (and not before) the interrupt delivery.
5665 */
5666 if (!rc)
5667 vcpu->guest_debug &= ~KVM_GUESTDBG_EXIT_PENDING;
5668
5669 return rc;
5670 }
5671
kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu * vcpu,struct kvm_pv_cmd * cmd)5672 static int kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu *vcpu,
5673 struct kvm_pv_cmd *cmd)
5674 {
5675 struct kvm_s390_pv_dmp dmp;
5676 void *data;
5677 int ret;
5678
5679 /* Dump initialization is a prerequisite */
5680 if (!vcpu->kvm->arch.pv.dumping)
5681 return -EINVAL;
5682
5683 if (copy_from_user(&dmp, (__u8 __user *)cmd->data, sizeof(dmp)))
5684 return -EFAULT;
5685
5686 /* We only handle this subcmd right now */
5687 if (dmp.subcmd != KVM_PV_DUMP_CPU)
5688 return -EINVAL;
5689
5690 /* CPU dump length is the same as create cpu storage donation. */
5691 if (dmp.buff_len != uv_info.guest_cpu_stor_len)
5692 return -EINVAL;
5693
5694 data = kvzalloc(uv_info.guest_cpu_stor_len, GFP_KERNEL);
5695 if (!data)
5696 return -ENOMEM;
5697
5698 ret = kvm_s390_pv_dump_cpu(vcpu, data, &cmd->rc, &cmd->rrc);
5699
5700 VCPU_EVENT(vcpu, 3, "PROTVIRT DUMP CPU %d rc %x rrc %x",
5701 vcpu->vcpu_id, cmd->rc, cmd->rrc);
5702
5703 if (ret)
5704 ret = -EINVAL;
5705
5706 /* On success copy over the dump data */
5707 if (!ret && copy_to_user((__u8 __user *)dmp.buff_addr, data, uv_info.guest_cpu_stor_len))
5708 ret = -EFAULT;
5709
5710 kvfree(data);
5711 return ret;
5712 }
5713
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5714 long kvm_arch_vcpu_ioctl(struct file *filp,
5715 unsigned int ioctl, unsigned long arg)
5716 {
5717 struct kvm_vcpu *vcpu = filp->private_data;
5718 void __user *argp = (void __user *)arg;
5719 int idx;
5720 long r;
5721 u16 rc, rrc;
5722
5723 vcpu_load(vcpu);
5724
5725 switch (ioctl) {
5726 case KVM_S390_STORE_STATUS:
5727 idx = srcu_read_lock(&vcpu->kvm->srcu);
5728 r = kvm_s390_store_status_unloaded(vcpu, arg);
5729 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5730 break;
5731 case KVM_S390_SET_INITIAL_PSW: {
5732 psw_t psw;
5733
5734 r = -EFAULT;
5735 if (copy_from_user(&psw, argp, sizeof(psw)))
5736 break;
5737 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
5738 break;
5739 }
5740 case KVM_S390_CLEAR_RESET:
5741 r = 0;
5742 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
5743 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5744 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5745 UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
5746 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
5747 rc, rrc);
5748 }
5749 break;
5750 case KVM_S390_INITIAL_RESET:
5751 r = 0;
5752 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
5753 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5754 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5755 UVC_CMD_CPU_RESET_INITIAL,
5756 &rc, &rrc);
5757 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
5758 rc, rrc);
5759 }
5760 break;
5761 case KVM_S390_NORMAL_RESET:
5762 r = 0;
5763 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
5764 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5765 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5766 UVC_CMD_CPU_RESET, &rc, &rrc);
5767 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
5768 rc, rrc);
5769 }
5770 break;
5771 case KVM_SET_ONE_REG:
5772 case KVM_GET_ONE_REG: {
5773 struct kvm_one_reg reg;
5774 r = -EINVAL;
5775 if (kvm_s390_pv_cpu_is_protected(vcpu))
5776 break;
5777 r = -EFAULT;
5778 if (copy_from_user(®, argp, sizeof(reg)))
5779 break;
5780 if (ioctl == KVM_SET_ONE_REG)
5781 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
5782 else
5783 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
5784 break;
5785 }
5786 #ifdef CONFIG_KVM_S390_UCONTROL
5787 case KVM_S390_UCAS_MAP: {
5788 struct kvm_s390_ucas_mapping ucasmap;
5789
5790 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
5791 r = -EFAULT;
5792 break;
5793 }
5794
5795 if (!kvm_is_ucontrol(vcpu->kvm)) {
5796 r = -EINVAL;
5797 break;
5798 }
5799
5800 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
5801 ucasmap.vcpu_addr, ucasmap.length);
5802 break;
5803 }
5804 case KVM_S390_UCAS_UNMAP: {
5805 struct kvm_s390_ucas_mapping ucasmap;
5806
5807 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
5808 r = -EFAULT;
5809 break;
5810 }
5811
5812 if (!kvm_is_ucontrol(vcpu->kvm)) {
5813 r = -EINVAL;
5814 break;
5815 }
5816
5817 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
5818 ucasmap.length);
5819 break;
5820 }
5821 #endif
5822 case KVM_S390_VCPU_FAULT: {
5823 idx = srcu_read_lock(&vcpu->kvm->srcu);
5824 r = vcpu_dat_fault_handler(vcpu, arg, 0);
5825 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5826 break;
5827 }
5828 case KVM_ENABLE_CAP:
5829 {
5830 struct kvm_enable_cap cap;
5831 r = -EFAULT;
5832 if (copy_from_user(&cap, argp, sizeof(cap)))
5833 break;
5834 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5835 break;
5836 }
5837 case KVM_S390_MEM_OP: {
5838 struct kvm_s390_mem_op mem_op;
5839
5840 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
5841 r = kvm_s390_vcpu_memsida_op(vcpu, &mem_op);
5842 else
5843 r = -EFAULT;
5844 break;
5845 }
5846 case KVM_S390_SET_IRQ_STATE: {
5847 struct kvm_s390_irq_state irq_state;
5848
5849 r = -EFAULT;
5850 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5851 break;
5852 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
5853 irq_state.len == 0 ||
5854 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
5855 r = -EINVAL;
5856 break;
5857 }
5858 /* do not use irq_state.flags, it will break old QEMUs */
5859 r = kvm_s390_set_irq_state(vcpu,
5860 (void __user *) irq_state.buf,
5861 irq_state.len);
5862 break;
5863 }
5864 case KVM_S390_GET_IRQ_STATE: {
5865 struct kvm_s390_irq_state irq_state;
5866
5867 r = -EFAULT;
5868 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5869 break;
5870 if (irq_state.len == 0) {
5871 r = -EINVAL;
5872 break;
5873 }
5874 /* do not use irq_state.flags, it will break old QEMUs */
5875 r = kvm_s390_get_irq_state(vcpu,
5876 (__u8 __user *) irq_state.buf,
5877 irq_state.len);
5878 break;
5879 }
5880 case KVM_S390_PV_CPU_COMMAND: {
5881 struct kvm_pv_cmd cmd;
5882
5883 r = -EINVAL;
5884 if (!is_prot_virt_host())
5885 break;
5886
5887 r = -EFAULT;
5888 if (copy_from_user(&cmd, argp, sizeof(cmd)))
5889 break;
5890
5891 r = -EINVAL;
5892 if (cmd.flags)
5893 break;
5894
5895 /* We only handle this cmd right now */
5896 if (cmd.cmd != KVM_PV_DUMP)
5897 break;
5898
5899 r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd);
5900
5901 /* Always copy over UV rc / rrc data */
5902 if (copy_to_user((__u8 __user *)argp, &cmd.rc,
5903 sizeof(cmd.rc) + sizeof(cmd.rrc)))
5904 r = -EFAULT;
5905 break;
5906 }
5907 default:
5908 r = -ENOTTY;
5909 }
5910
5911 vcpu_put(vcpu);
5912 return r;
5913 }
5914
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)5915 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5916 {
5917 #ifdef CONFIG_KVM_S390_UCONTROL
5918 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
5919 && (kvm_is_ucontrol(vcpu->kvm))) {
5920 vmf->page = virt_to_page(vcpu->arch.sie_block);
5921 get_page(vmf->page);
5922 return 0;
5923 }
5924 #endif
5925 return VM_FAULT_SIGBUS;
5926 }
5927
kvm_arch_irqchip_in_kernel(struct kvm * kvm)5928 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
5929 {
5930 return true;
5931 }
5932
5933 /* Section: memory related */
kvm_arch_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)5934 int kvm_arch_prepare_memory_region(struct kvm *kvm,
5935 const struct kvm_memory_slot *old,
5936 struct kvm_memory_slot *new,
5937 enum kvm_mr_change change)
5938 {
5939 gpa_t size;
5940
5941 if (kvm_is_ucontrol(kvm) && new->id < KVM_USER_MEM_SLOTS)
5942 return -EINVAL;
5943
5944 /* When we are protected, we should not change the memory slots */
5945 if (kvm_s390_pv_get_handle(kvm))
5946 return -EINVAL;
5947
5948 if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) {
5949 /*
5950 * A few sanity checks. We can have memory slots which have to be
5951 * located/ended at a segment boundary (1MB). The memory in userland is
5952 * ok to be fragmented into various different vmas. It is okay to mmap()
5953 * and munmap() stuff in this slot after doing this call at any time
5954 */
5955
5956 if (new->userspace_addr & 0xffffful)
5957 return -EINVAL;
5958
5959 size = new->npages * PAGE_SIZE;
5960 if (size & 0xffffful)
5961 return -EINVAL;
5962
5963 if ((new->base_gfn * PAGE_SIZE) + size > kvm->arch.mem_limit)
5964 return -EINVAL;
5965 }
5966
5967 if (!kvm->arch.migration_mode)
5968 return 0;
5969
5970 /*
5971 * Turn off migration mode when:
5972 * - userspace creates a new memslot with dirty logging off,
5973 * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and
5974 * dirty logging is turned off.
5975 * Migration mode expects dirty page logging being enabled to store
5976 * its dirty bitmap.
5977 */
5978 if (change != KVM_MR_DELETE &&
5979 !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
5980 WARN(kvm_s390_vm_stop_migration(kvm),
5981 "Failed to stop migration mode");
5982
5983 return 0;
5984 }
5985
kvm_arch_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)5986 void kvm_arch_commit_memory_region(struct kvm *kvm,
5987 struct kvm_memory_slot *old,
5988 const struct kvm_memory_slot *new,
5989 enum kvm_mr_change change)
5990 {
5991 int rc = 0;
5992
5993 if (kvm_is_ucontrol(kvm))
5994 return;
5995
5996 switch (change) {
5997 case KVM_MR_DELETE:
5998 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
5999 old->npages * PAGE_SIZE);
6000 break;
6001 case KVM_MR_MOVE:
6002 rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
6003 old->npages * PAGE_SIZE);
6004 if (rc)
6005 break;
6006 fallthrough;
6007 case KVM_MR_CREATE:
6008 rc = gmap_map_segment(kvm->arch.gmap, new->userspace_addr,
6009 new->base_gfn * PAGE_SIZE,
6010 new->npages * PAGE_SIZE);
6011 break;
6012 case KVM_MR_FLAGS_ONLY:
6013 break;
6014 default:
6015 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
6016 }
6017 if (rc)
6018 pr_warn("failed to commit memory region\n");
6019 return;
6020 }
6021
nonhyp_mask(int i)6022 static inline unsigned long nonhyp_mask(int i)
6023 {
6024 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
6025
6026 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
6027 }
6028
kvm_s390_init(void)6029 static int __init kvm_s390_init(void)
6030 {
6031 int i, r;
6032
6033 if (!sclp.has_sief2) {
6034 pr_info("SIE is not available\n");
6035 return -ENODEV;
6036 }
6037
6038 if (nested && hpage) {
6039 pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
6040 return -EINVAL;
6041 }
6042
6043 for (i = 0; i < 16; i++)
6044 kvm_s390_fac_base[i] |=
6045 stfle_fac_list[i] & nonhyp_mask(i);
6046
6047 r = __kvm_s390_init();
6048 if (r)
6049 return r;
6050
6051 r = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
6052 if (r) {
6053 __kvm_s390_exit();
6054 return r;
6055 }
6056 return 0;
6057 }
6058
kvm_s390_exit(void)6059 static void __exit kvm_s390_exit(void)
6060 {
6061 kvm_exit();
6062
6063 __kvm_s390_exit();
6064 }
6065
6066 module_init(kvm_s390_init);
6067 module_exit(kvm_s390_exit);
6068
6069 /*
6070 * Enable autoloading of the kvm module.
6071 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
6072 * since x86 takes a different approach.
6073 */
6074 #include <linux/miscdevice.h>
6075 MODULE_ALIAS_MISCDEV(KVM_MINOR);
6076 MODULE_ALIAS("devname:kvm");
6077