1 // SPDX-License-Identifier: GPL-2.0
3  * Hosting Protected Virtual Machines
21 #include "kvm-s390.h"
25 	lockdep_assert_held(&kvm->lock);  in kvm_s390_pv_is_protected()
32 	lockdep_assert_held(&vcpu->mutex);  in kvm_s390_pv_cpu_is_protected()
38  * kvm_s390_pv_make_secure() - make one guest page secure
43  * Context: needs to be called with kvm->srcu held.
50 	lockdep_assert_held(&kvm->srcu);  in kvm_s390_pv_make_secure()
54 		return -EFAULT;  in kvm_s390_pv_make_secure()
55 	return make_hva_secure(kvm->mm, vmaddr, uvcb);  in kvm_s390_pv_make_secure()
71  * kvm_s390_pv_destroy_page() - Destroy a guest page.
86 	mmap_read_lock(kvm->mm);  in kvm_s390_pv_destroy_page()
91 	mmap_read_unlock(kvm->mm);  in kvm_s390_pv_destroy_page()
96  * struct pv_vm_to_be_destroyed - Represents a protected VM that needs to
100  * @old_gmap_table: the gmap table of the leftover protected VM
101  * @handle: the handle of the leftover protected VM
102  * @stor_var: pointer to the variable storage of the leftover protected VM
103  * @stor_base: address of the base storage of the leftover protected VM
105  * Represents a protected VM that is still registered with the Ultravisor,
120 	kvm->arch.pv.handle = 0;  in kvm_s390_clear_pv_state()
121 	kvm->arch.pv.guest_len = 0;  in kvm_s390_clear_pv_state()
122 	kvm->arch.pv.stor_base = 0;  in kvm_s390_clear_pv_state()
123 	kvm->arch.pv.stor_var = NULL;  in kvm_s390_clear_pv_state()
135 	KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT DESTROY VCPU %d: rc %x rrc %x",  in kvm_s390_pv_destroy_cpu()
136 		     vcpu->vcpu_id, *rc, *rrc);  in kvm_s390_pv_destroy_cpu()
141 		free_pages(vcpu->arch.pv.stor_base,  in kvm_s390_pv_destroy_cpu()
144 	free_page((unsigned long)sida_addr(vcpu->arch.sie_block));  in kvm_s390_pv_destroy_cpu()
145 	vcpu->arch.sie_block->pv_handle_cpu = 0;  in kvm_s390_pv_destroy_cpu()
146 	vcpu->arch.sie_block->pv_handle_config = 0;  in kvm_s390_pv_destroy_cpu()
147 	memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv));  in kvm_s390_pv_destroy_cpu()
148 	vcpu->arch.sie_block->sdf = 0;  in kvm_s390_pv_destroy_cpu()
154 	vcpu->arch.sie_block->gbea = 1;  in kvm_s390_pv_destroy_cpu()
170 		return -EINVAL;  in kvm_s390_pv_create_cpu()
172 	vcpu->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT,  in kvm_s390_pv_create_cpu()
174 	if (!vcpu->arch.pv.stor_base)  in kvm_s390_pv_create_cpu()
175 		return -ENOMEM;  in kvm_s390_pv_create_cpu()
178 	uvcb.guest_handle = kvm_s390_pv_get_handle(vcpu->kvm);  in kvm_s390_pv_create_cpu()
179 	uvcb.num = vcpu->arch.sie_block->icpua;  in kvm_s390_pv_create_cpu()
180 	uvcb.state_origin = virt_to_phys(vcpu->arch.sie_block);  in kvm_s390_pv_create_cpu()
181 	uvcb.stor_origin = virt_to_phys((void *)vcpu->arch.pv.stor_base);  in kvm_s390_pv_create_cpu()
186 		free_pages(vcpu->arch.pv.stor_base,  in kvm_s390_pv_create_cpu()
188 		return -ENOMEM;  in kvm_s390_pv_create_cpu()
190 	vcpu->arch.sie_block->sidad = virt_to_phys(sida_addr);  in kvm_s390_pv_create_cpu()
195 	KVM_UV_EVENT(vcpu->kvm, 3,  in kvm_s390_pv_create_cpu()
197 		     vcpu->vcpu_id, uvcb.cpu_handle, uvcb.header.rc,  in kvm_s390_pv_create_cpu()
204 		return -EIO;  in kvm_s390_pv_create_cpu()
208 	vcpu->arch.pv.handle = uvcb.cpu_handle;  in kvm_s390_pv_create_cpu()
209 	vcpu->arch.sie_block->pv_handle_cpu = uvcb.cpu_handle;  in kvm_s390_pv_create_cpu()
210 	vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_get_handle(vcpu->kvm);  in kvm_s390_pv_create_cpu()
211 	vcpu->arch.sie_block->sdf = 2;  in kvm_s390_pv_create_cpu()
219 	vfree(kvm->arch.pv.stor_var);  in kvm_s390_pv_dealloc_vm()
220 	free_pages(kvm->arch.pv.stor_base,  in kvm_s390_pv_dealloc_vm()
231 	kvm->arch.pv.stor_var = NULL;  in kvm_s390_pv_alloc_vm()
232 	kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, get_order(base));  in kvm_s390_pv_alloc_vm()
233 	if (!kvm->arch.pv.stor_base)  in kvm_s390_pv_alloc_vm()
234 		return -ENOMEM;  in kvm_s390_pv_alloc_vm()
242 	mutex_lock(&kvm->slots_lock);  in kvm_s390_pv_alloc_vm()
244 	mutex_unlock(&kvm->slots_lock);  in kvm_s390_pv_alloc_vm()
246 	kvm->arch.pv.guest_len = npages * PAGE_SIZE;  in kvm_s390_pv_alloc_vm()
251 	kvm->arch.pv.stor_var = vzalloc(vlen);  in kvm_s390_pv_alloc_vm()
252 	if (!kvm->arch.pv.stor_var)  in kvm_s390_pv_alloc_vm()
258 	return -ENOMEM;  in kvm_s390_pv_alloc_vm()
262  * kvm_s390_pv_dispose_one_leftover - Clean up one leftover protected VM.
263  * @kvm: the KVM that was associated with this leftover protected VM
264  * @leftover: details about the leftover protected VM that needs a clean up
268  * Destroy one leftover protected VM.
269  * On success, kvm->mm->context.protected_count will be decremented atomically
280 	/* It used the destroy-fast UVC, nothing left to do here */  in kvm_s390_pv_dispose_one_leftover()
281 	if (!leftover->handle)  in kvm_s390_pv_dispose_one_leftover()
283 	cc = uv_cmd_nodata(leftover->handle, UVC_CMD_DESTROY_SEC_CONF, rc, rrc);  in kvm_s390_pv_dispose_one_leftover()
291 	 * This can only happen in case of a serious KVM or hardware bug; it  in kvm_s390_pv_dispose_one_leftover()
294 	free_pages(leftover->stor_base, get_order(uv_info.guest_base_stor_len));  in kvm_s390_pv_dispose_one_leftover()
295 	free_pages(leftover->old_gmap_table, CRST_ALLOC_ORDER);  in kvm_s390_pv_dispose_one_leftover()
296 	vfree(leftover->stor_var);  in kvm_s390_pv_dispose_one_leftover()
298 	atomic_dec(&kvm->mm->context.protected_count);  in kvm_s390_pv_dispose_one_leftover()
303  * kvm_s390_destroy_lower_2g - Destroy the first 2GB of protected guest memory.
307  * The CPUs of the protected VM need to be destroyed beforehand.
316 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvm_s390_destroy_lower_2g()
321 	while (slot && slot->base_gfn < pages_2g) {  in kvm_s390_destroy_lower_2g()
322 		len = min_t(u64, slot->npages, pages_2g - slot->base_gfn) * PAGE_SIZE;  in kvm_s390_destroy_lower_2g()
323 		s390_uv_destroy_range(kvm->mm, slot->userspace_addr, slot->userspace_addr + len);  in kvm_s390_destroy_lower_2g()
325 		slot = gfn_to_memslot(kvm, slot->base_gfn + slot->npages);  in kvm_s390_destroy_lower_2g()
328 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvm_s390_destroy_lower_2g()
345 	WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);  in kvm_s390_pv_deinit_vm_fast()
354 	return cc ? -EIO : 0;  in kvm_s390_pv_deinit_vm_fast()
363  * kvm_s390_pv_set_aside - Set aside a protected VM for later teardown.
368  * Set aside the protected VM for a subsequent teardown. The VM will be able
369  * to continue immediately as a non-secure VM, and the information needed to
370  * properly tear down the protected VM is set aside. If another protected VM
373  * The CPUs of the protected VM need to be destroyed beforehand.
375  * Context: kvm->lock needs to be held
377  * Return: 0 in case of success, -EINVAL if another protected VM was already set
378  * aside, -ENOMEM if the system ran out of memory.
385 	lockdep_assert_held(&kvm->lock);  in kvm_s390_pv_set_aside()
387 	 * If another protected VM was already prepared for teardown, refuse.  in kvm_s390_pv_set_aside()
390 	if (kvm->arch.pv.set_aside)  in kvm_s390_pv_set_aside()
391 		return -EINVAL;  in kvm_s390_pv_set_aside()
394 	if ((kvm->arch.gmap->asce & _ASCE_TYPE_MASK) == _ASCE_TYPE_SEGMENT)  in kvm_s390_pv_set_aside()
395 		return -EINVAL;  in kvm_s390_pv_set_aside()
399 		return -ENOMEM;  in kvm_s390_pv_set_aside()
404 		priv->stor_var = kvm->arch.pv.stor_var;  in kvm_s390_pv_set_aside()
405 		priv->stor_base = kvm->arch.pv.stor_base;  in kvm_s390_pv_set_aside()
406 		priv->handle = kvm_s390_pv_get_handle(kvm);  in kvm_s390_pv_set_aside()
407 		priv->old_gmap_table = (unsigned long)kvm->arch.gmap->table;  in kvm_s390_pv_set_aside()
408 		WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);  in kvm_s390_pv_set_aside()
409 		if (s390_replace_asce(kvm->arch.gmap))  in kvm_s390_pv_set_aside()
410 			res = -ENOMEM;  in kvm_s390_pv_set_aside()
420 	kvm->arch.pv.set_aside = priv;  in kvm_s390_pv_set_aside()
428  * kvm_s390_pv_deinit_vm - Deinitialize the current protected VM
429  * @kvm: the KVM whose protected VM needs to be deinitialized
433  * Deinitialize the current protected VM. This function will destroy and
434  * cleanup the current protected VM, but it will not cleanup the guest
435  * memory. This function should only be called when the protected VM has
442  * Context: kvm->lock needs to be held
444  * Return: 0 in case of success, otherwise -EIO
452 	WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);  in kvm_s390_pv_deinit_vm()
454 		atomic_dec(&kvm->mm->context.protected_count);  in kvm_s390_pv_deinit_vm()
458 		s390_replace_asce(kvm->arch.gmap);  in kvm_s390_pv_deinit_vm()
463 	return cc ? -EIO : 0;  in kvm_s390_pv_deinit_vm()
467  * kvm_s390_pv_deinit_cleanup_all - Clean up all protected VMs associated
473  * This function will clean up all protected VMs associated with a KVM.
477  * Context: kvm->lock needs to be held unless being called from
480  * Return: 0 if all VMs are successfully cleaned up, otherwise -EIO
493 	if (!atomic_inc_not_zero(&kvm->mm->context.protected_count))  in kvm_s390_pv_deinit_cleanup_all()
497 	/* If the current VM is protected, destroy it */  in kvm_s390_pv_deinit_cleanup_all()
503 	/* If a previous protected VM was set aside, put it in the need_cleanup list */  in kvm_s390_pv_deinit_cleanup_all()
504 	if (kvm->arch.pv.set_aside) {  in kvm_s390_pv_deinit_cleanup_all()
505 		list_add(kvm->arch.pv.set_aside, &kvm->arch.pv.need_cleanup);  in kvm_s390_pv_deinit_cleanup_all()
506 		kvm->arch.pv.set_aside = NULL;  in kvm_s390_pv_deinit_cleanup_all()
509 	/* Cleanup all protected VMs in the need_cleanup list */  in kvm_s390_pv_deinit_cleanup_all()
510 	while (!list_empty(&kvm->arch.pv.need_cleanup)) {  in kvm_s390_pv_deinit_cleanup_all()
511 		cur = list_first_entry(&kvm->arch.pv.need_cleanup, typeof(*cur), list);  in kvm_s390_pv_deinit_cleanup_all()
525 		list_del(&cur->list);  in kvm_s390_pv_deinit_cleanup_all()
534 	if (need_zap && mmget_not_zero(kvm->mm)) {  in kvm_s390_pv_deinit_cleanup_all()
535 		s390_uv_destroy_range(kvm->mm, 0, TASK_SIZE);  in kvm_s390_pv_deinit_cleanup_all()
536 		mmput(kvm->mm);  in kvm_s390_pv_deinit_cleanup_all()
540 	atomic_dec(&kvm->mm->context.protected_count);  in kvm_s390_pv_deinit_cleanup_all()
541 	return cc ? -EIO : 0;  in kvm_s390_pv_deinit_cleanup_all()
545  * kvm_s390_pv_deinit_aside_vm - Teardown a previously set aside protected VM.
546  * @kvm: the VM previously associated with the protected VM
550  * Tear down the protected VM that had been previously prepared for teardown
554  * Context: kvm->lock must not be held.
556  * Return: 0 in case of success, -EINVAL if no protected VM had been
557  * prepared for asynchronous teardowm, -EIO in case of other errors.
564 	lockdep_assert_not_held(&kvm->lock);  in kvm_s390_pv_deinit_aside_vm()
565 	mutex_lock(&kvm->lock);  in kvm_s390_pv_deinit_aside_vm()
566 	p = kvm->arch.pv.set_aside;  in kvm_s390_pv_deinit_aside_vm()
567 	kvm->arch.pv.set_aside = NULL;  in kvm_s390_pv_deinit_aside_vm()
568 	mutex_unlock(&kvm->lock);  in kvm_s390_pv_deinit_aside_vm()
570 		return -EINVAL;  in kvm_s390_pv_deinit_aside_vm()
573 	if (s390_uv_destroy_range_interruptible(kvm->mm, 0, TASK_SIZE_MAX))  in kvm_s390_pv_deinit_aside_vm()
576 		ret = -EIO;  in kvm_s390_pv_deinit_aside_vm()
585 		mutex_lock(&kvm->lock);  in kvm_s390_pv_deinit_aside_vm()
586 		list_add(&p->list, &kvm->arch.pv.need_cleanup);  in kvm_s390_pv_deinit_aside_vm()
587 		mutex_unlock(&kvm->lock);  in kvm_s390_pv_deinit_aside_vm()
627 	/* Add the notifier only once. No races because we hold kvm->lock */  in kvm_s390_pv_init_vm()
628 	if (kvm->arch.pv.mmu_notifier.ops != &kvm_s390_pv_mmu_notifier_ops) {  in kvm_s390_pv_init_vm()
630 		kvm->arch.pv.mmu_notifier.ops = &kvm_s390_pv_mmu_notifier_ops;  in kvm_s390_pv_init_vm()
631 		ret = mmu_notifier_register(&kvm->arch.pv.mmu_notifier, kvm->mm);  in kvm_s390_pv_init_vm()
633 			kvm->arch.pv.mmu_notifier.ops = NULL;  in kvm_s390_pv_init_vm()
644 	uvcb.guest_stor_len = kvm->arch.pv.guest_len;  in kvm_s390_pv_init_vm()
645 	uvcb.guest_asce = kvm->arch.gmap->asce;  in kvm_s390_pv_init_vm()
646 	uvcb.guest_sca = virt_to_phys(kvm->arch.sca);  in kvm_s390_pv_init_vm()
648 		virt_to_phys((void *)kvm->arch.pv.stor_base);  in kvm_s390_pv_init_vm()
649 	uvcb.conf_virt_stor_origin = (u64)kvm->arch.pv.stor_var;  in kvm_s390_pv_init_vm()
650 	uvcb.flags.ap_allow_instr = kvm->arch.model.uv_feat_guest.ap;  in kvm_s390_pv_init_vm()
651 	uvcb.flags.ap_instr_intr = kvm->arch.model.uv_feat_guest.ap_intr;  in kvm_s390_pv_init_vm()
660 	kvm->arch.pv.handle = uvcb.guest_handle;  in kvm_s390_pv_init_vm()
662 	atomic_inc(&kvm->mm->context.protected_count);  in kvm_s390_pv_init_vm()
667 			atomic_dec(&kvm->mm->context.protected_count);  in kvm_s390_pv_init_vm()
670 		return -EIO;  in kvm_s390_pv_init_vm()
672 	kvm->arch.gmap->guest_handle = uvcb.guest_handle;  in kvm_s390_pv_init_vm()
692 	return cc ? -EINVAL : 0;  in kvm_s390_pv_set_sec_parms()
713 	if (ret == -ENXIO) {  in unpack_one()
714 		mmap_read_lock(kvm->mm);  in unpack_one()
717 			ret = -EFAULT;  in unpack_one()
719 			ret = fixup_user_fault(kvm->mm, vmaddr, FAULT_FLAG_WRITE, &unlocked);  in unpack_one()
721 				ret = __gmap_link(kvm->arch.gmap, addr, vmaddr);  in unpack_one()
723 		mmap_read_unlock(kvm->mm);  in unpack_one()
725 			return -EAGAIN;  in unpack_one()
729 	if (ret && ret != -EAGAIN)  in unpack_one()
742 		return -EINVAL;  in kvm_s390_pv_unpack()
747 	guard(srcu)(&kvm->srcu);  in kvm_s390_pv_unpack()
751 		if (ret == -EAGAIN) {  in kvm_s390_pv_unpack()
778 	KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT SET CPU %d STATE %d rc %x rrc %x",  in kvm_s390_pv_set_cpu_state()
779 		     vcpu->vcpu_id, state, uvcb.header.rc, uvcb.header.rrc);  in kvm_s390_pv_set_cpu_state()
781 		return -EINVAL;  in kvm_s390_pv_set_cpu_state()
790 		.cpu_handle = vcpu->arch.pv.handle,  in kvm_s390_pv_dump_cpu()
822  * Context: kvm->lock needs to be held
826  *  -ENOMEM if allocating the cache fails
827  *  -EINVAL if gaddr is not aligned to 1MB
828  *  -EINVAL if buff_user_len is not aligned to uv_info.conf_dump_storage_state_len
829  *  -EINVAL if the UV call fails, rc and rrc will be set in this case
830  *  -EFAULT if copying the result to buff_user failed
838 		.config_handle = kvm->arch.pv.handle,  in kvm_s390_pv_dump_stor_state()
848 	ret = -EINVAL;  in kvm_s390_pv_dump_stor_state()
868 	ret = -ENOMEM;  in kvm_s390_pv_dump_stor_state()
883 			ret = -EINVAL;  in kvm_s390_pv_dump_stor_state()
889 		buff_user_len -= increment_len;  in kvm_s390_pv_dump_stor_state()
895 				ret = -EFAULT;  in kvm_s390_pv_dump_stor_state()
932  * Context: kvm->lock needs to be held
936  *  -ENOMEM if allocating the completion buffer fails
937  *  -EINVAL if the UV call fails, rc and rrc will be set in this case
938  *  -EFAULT if copying the result to buff_user failed
954 		return -ENOMEM;  in kvm_s390_pv_dump_complete()
969 		kvm->arch.pv.dumping = false;  in kvm_s390_pv_dump_complete()
973 			ret = -EFAULT;  in kvm_s390_pv_dump_complete()
976 	/* If the UVC returned an error, translate it to -EINVAL */  in kvm_s390_pv_dump_complete()
978 		ret = -EINVAL;  in kvm_s390_pv_dump_complete()