xref: /linux/arch/s390/kvm/pv.c (revision 42546fc642e929de07459ff839a9f43a653ffb4e)

// SPDX-License-Identifier: GPL-2.0
/*
 * Hosting Protected Virtual Machines
 *
 * Copyright IBM Corp. 2019, 2020
 *    Author(s): Janosch Frank <frankja@linux.ibm.com>
 */

#include <linux/export.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/minmax.h>
#include <linux/pagemap.h>
#include <linux/sched/signal.h>
#include <asm/uv.h>
#include <asm/mman.h>
#include <linux/pagewalk.h>
#include <linux/sched/mm.h>
#include <linux/mmu_notifier.h>
#include <asm/gmap_helpers.h>
#include "kvm-s390.h"
#include "dat.h"
#include "gaccess.h"
#include "gmap.h"
#include "faultin.h"

bool kvm_s390_pv_is_protected(struct kvm *kvm)
{
	lockdep_assert_held(&kvm->lock);
	return !!kvm_s390_pv_get_handle(kvm);
}
EXPORT_SYMBOL_GPL(kvm_s390_pv_is_protected);

bool kvm_s390_pv_cpu_is_protected(struct kvm_vcpu *vcpu)
{
	lockdep_assert_held(&vcpu->mutex);
	return !!kvm_s390_pv_cpu_get_handle(vcpu);
}
EXPORT_SYMBOL_GPL(kvm_s390_pv_cpu_is_protected);

/**
 * should_export_before_import() - Determine whether an export is needed
 * before an import-like operation.
 * @uvcb: The Ultravisor control block of the UVC to be performed.
 * @mm: The mm of the process.
 *
 * Returns whether an export is needed before every import-like operation.
 * This is needed for shared pages, which don't trigger a secure storage
 * exception when accessed from a different guest.
 *
 * Although considered as one, the Unpin Page UVC is not an actual import,
 * so it is not affected.
 *
 * No export is needed also when there is only one protected VM, because the
 * page cannot belong to the wrong VM in that case (there is no "other VM"
 * it can belong to).
 *
 * Return: %true if an export is needed before every import, otherwise %false.
 */
static bool should_export_before_import(struct uv_cb_header *uvcb, struct mm_struct *mm)
{
	/*
	 * The misc feature indicates, among other things, that importing a
	 * shared page from a different protected VM will automatically also
	 * transfer its ownership.
	 */
	if (uv_has_feature(BIT_UV_FEAT_MISC))
		return false;
	if (uvcb->cmd == UVC_CMD_UNPIN_PAGE_SHARED)
		return false;
	return atomic_read(&mm->context.protected_count) > 1;
}

struct pv_make_secure {
	void *uvcb;
	struct folio *folio;
	struct kvm *kvm;
	int rc;
	bool needs_export;
};

static int __kvm_s390_pv_make_secure(struct guest_fault *f, struct folio *folio)
{
	struct pv_make_secure *priv = f->priv;
	int rc;

	if (priv->needs_export)
		uv_convert_from_secure(folio_to_phys(folio));

	if (folio_test_hugetlb(folio))
		return -EFAULT;
	if (folio_test_large(folio))
		return -E2BIG;

	if (!f->page)
		folio_get(folio);
	rc = __make_folio_secure(folio, priv->uvcb);
	if (!f->page)
		folio_put(folio);

	return rc;
}

static void _kvm_s390_pv_make_secure(struct guest_fault *f)
{
	struct pv_make_secure *priv = f->priv;
	struct folio *folio;
	spinlock_t *ptl;	/* pte lock from try_get_locked_pte() */
	pte_t *ptep;

	folio = pfn_folio(f->pfn);
	priv->rc = -EAGAIN;

	if (!mmap_read_trylock(priv->kvm->mm))
		return;

	ptep = try_get_locked_pte(priv->kvm->mm, gfn_to_hva(priv->kvm, f->gfn), &ptl);
	if (IS_ERR_VALUE(ptep)) {
		priv->rc = PTR_ERR(ptep);
		goto out;
	}

	if (folio_trylock(folio)) {
		priv->rc = __kvm_s390_pv_make_secure(f, folio);
		if (priv->rc == -E2BIG || priv->rc == -EBUSY) {
			priv->folio = folio;
			folio_get(folio);
		}
		folio_unlock(folio);
	}

	if (ptep)
		pte_unmap_unlock(ptep, ptl);
out:
	mmap_read_unlock(priv->kvm->mm);
}

/**
 * kvm_s390_pv_make_secure() - make one guest page secure
 * @kvm: the guest
 * @gaddr: the guest address that needs to be made secure
 * @uvcb: the UVCB specifying which operation needs to be performed
 *
 * Context: needs to be called with kvm->srcu held.
 * Return: 0 on success, < 0 in case of error.
 */
int kvm_s390_pv_make_secure(struct kvm *kvm, unsigned long gaddr, void *uvcb)
{
	struct pv_make_secure priv = { .uvcb = uvcb, .kvm = kvm, };
	struct guest_fault f = {
		.write_attempt = true,
		.gfn = gpa_to_gfn(gaddr),
		.callback = _kvm_s390_pv_make_secure,
		.priv = &priv,
	};
	int rc;

	lockdep_assert_held(&kvm->srcu);

	priv.needs_export = should_export_before_import(uvcb, kvm->mm);

	scoped_guard(mutex, &kvm->arch.pv.import_lock) {
		rc = kvm_s390_faultin_gfn(NULL, kvm, &f);

		if (!rc) {
			rc = priv.rc;
			if (priv.folio) {
				rc = s390_wiggle_split_folio(kvm->mm, priv.folio);
				if (!rc)
					rc = -EAGAIN;
			}
		}
	}
	if (priv.folio)
		folio_put(priv.folio);
	return rc;
}

int kvm_s390_pv_convert_to_secure(struct kvm *kvm, unsigned long gaddr)
{
	struct uv_cb_cts uvcb = {
		.header.cmd = UVC_CMD_CONV_TO_SEC_STOR,
		.header.len = sizeof(uvcb),
		.guest_handle = kvm_s390_pv_get_handle(kvm),
		.gaddr = gaddr,
	};

	return kvm_s390_pv_make_secure(kvm, gaddr, &uvcb);
}

/**
 * kvm_s390_pv_destroy_page() - Destroy a guest page.
 * @kvm: the guest
 * @gaddr: the guest address to destroy
 *
 * An attempt will be made to destroy the given guest page. If the attempt
 * fails, an attempt is made to export the page. If both attempts fail, an
 * appropriate error is returned.
 *
 * Context: may sleep.
 */
int kvm_s390_pv_destroy_page(struct kvm *kvm, unsigned long gaddr)
{
	struct page *page;
	int rc = 0;

	mmap_read_lock(kvm->mm);
	page = gfn_to_page(kvm, gpa_to_gfn(gaddr));
	if (page)
		rc = __kvm_s390_pv_destroy_page(page);
	kvm_release_page_clean(page);
	mmap_read_unlock(kvm->mm);
	return rc;
}

/**
 * struct pv_vm_to_be_destroyed - Represents a protected VM that needs to
 * be destroyed
 *
 * @list: list head for the list of leftover VMs
 * @old_gmap_table: the gmap table of the leftover protected VM
 * @handle: the handle of the leftover protected VM
 * @stor_var: pointer to the variable storage of the leftover protected VM
 * @stor_base: address of the base storage of the leftover protected VM
 *
 * Represents a protected VM that is still registered with the Ultravisor,
 * but which does not correspond any longer to an active KVM VM. It should
 * be destroyed at some point later, either asynchronously or when the
 * process terminates.
 */
struct pv_vm_to_be_destroyed {
	struct list_head list;
	unsigned long old_gmap_table;
	u64 handle;
	void *stor_var;
	unsigned long stor_base;
};

static void kvm_s390_clear_pv_state(struct kvm *kvm)
{
	kvm->arch.pv.handle = 0;
	kvm->arch.pv.guest_len = 0;
	kvm->arch.pv.stor_base = 0;
	kvm->arch.pv.stor_var = NULL;
}

int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc)
{
	int cc;

	if (!kvm_s390_pv_cpu_get_handle(vcpu))
		return 0;

	cc = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu), UVC_CMD_DESTROY_SEC_CPU, rc, rrc);

	KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT DESTROY VCPU %d: rc %x rrc %x",
		     vcpu->vcpu_id, *rc, *rrc);
	WARN_ONCE(cc, "protvirt destroy cpu failed rc %x rrc %x", *rc, *rrc);

	/* Intended memory leak for something that should never happen. */
	if (!cc)
		free_pages(vcpu->arch.pv.stor_base,
			   get_order(uv_info.guest_cpu_stor_len));

	free_page((unsigned long)sida_addr(vcpu->arch.sie_block));
	vcpu->arch.sie_block->pv_handle_cpu = 0;
	vcpu->arch.sie_block->pv_handle_config = 0;
	memset(&vcpu->arch.pv, 0, sizeof(vcpu->arch.pv));
	vcpu->arch.sie_block->sdf = 0;
	/*
	 * The sidad field (for sdf == 2) is now the gbea field (for sdf == 0).
	 * Use the reset value of gbea to avoid leaking the kernel pointer of
	 * the just freed sida.
	 */
	vcpu->arch.sie_block->gbea = 1;
	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);

	return cc ? EIO : 0;
}

int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc)
{
	struct uv_cb_csc uvcb = {
		.header.cmd = UVC_CMD_CREATE_SEC_CPU,
		.header.len = sizeof(uvcb),
	};
	void *sida_addr;
	int cc;

	if (kvm_s390_pv_cpu_get_handle(vcpu))
		return -EINVAL;

	vcpu->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT,
						   get_order(uv_info.guest_cpu_stor_len));
	if (!vcpu->arch.pv.stor_base)
		return -ENOMEM;

	/* Input */
	uvcb.guest_handle = kvm_s390_pv_get_handle(vcpu->kvm);
	uvcb.num = vcpu->arch.sie_block->icpua;
	uvcb.state_origin = virt_to_phys(vcpu->arch.sie_block);
	uvcb.stor_origin = virt_to_phys((void *)vcpu->arch.pv.stor_base);

	/* Alloc Secure Instruction Data Area Designation */
	sida_addr = (void *)__get_free_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
	if (!sida_addr) {
		free_pages(vcpu->arch.pv.stor_base,
			   get_order(uv_info.guest_cpu_stor_len));
		return -ENOMEM;
	}
	vcpu->arch.sie_block->sidad = virt_to_phys(sida_addr);

	cc = uv_call(0, (u64)&uvcb);
	*rc = uvcb.header.rc;
	*rrc = uvcb.header.rrc;
	KVM_UV_EVENT(vcpu->kvm, 3,
		     "PROTVIRT CREATE VCPU: cpu %d handle %llx rc %x rrc %x",
		     vcpu->vcpu_id, uvcb.cpu_handle, uvcb.header.rc,
		     uvcb.header.rrc);

	if (cc) {
		u16 dummy;

		kvm_s390_pv_destroy_cpu(vcpu, &dummy, &dummy);
		return -EIO;
	}

	/* Output */
	vcpu->arch.pv.handle = uvcb.cpu_handle;
	vcpu->arch.sie_block->pv_handle_cpu = uvcb.cpu_handle;
	vcpu->arch.sie_block->pv_handle_config = kvm_s390_pv_get_handle(vcpu->kvm);
	vcpu->arch.sie_block->sdf = 2;
	kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
	return 0;
}

/* only free resources when the destroy was successful */
static void kvm_s390_pv_dealloc_vm(struct kvm *kvm)
{
	vfree(kvm->arch.pv.stor_var);
	free_pages(kvm->arch.pv.stor_base,
		   get_order(uv_info.guest_base_stor_len));
	kvm_s390_clear_pv_state(kvm);
}

static int kvm_s390_pv_alloc_vm(struct kvm *kvm)
{
	unsigned long base = uv_info.guest_base_stor_len;
	unsigned long virt = uv_info.guest_virt_var_stor_len;
	unsigned long npages = 0, vlen = 0;

	kvm->arch.pv.stor_var = NULL;
	kvm->arch.pv.stor_base = __get_free_pages(GFP_KERNEL_ACCOUNT, get_order(base));
	if (!kvm->arch.pv.stor_base)
		return -ENOMEM;

	/*
	 * Calculate current guest storage for allocation of the
	 * variable storage, which is based on the length in MB.
	 *
	 * Slots are sorted by GFN
	 */
	mutex_lock(&kvm->slots_lock);
	npages = kvm_s390_get_gfn_end(kvm_memslots(kvm));
	mutex_unlock(&kvm->slots_lock);

	kvm->arch.pv.guest_len = npages * PAGE_SIZE;

	/* Allocate variable storage */
	vlen = ALIGN(virt * ((npages * PAGE_SIZE) / HPAGE_SIZE), PAGE_SIZE);
	vlen += uv_info.guest_virt_base_stor_len;
	kvm->arch.pv.stor_var = vzalloc(vlen);
	if (!kvm->arch.pv.stor_var)
		goto out_err;
	return 0;

out_err:
	kvm_s390_pv_dealloc_vm(kvm);
	return -ENOMEM;
}

/**
 * kvm_s390_pv_dispose_one_leftover - Clean up one leftover protected VM.
 * @kvm: the KVM that was associated with this leftover protected VM
 * @leftover: details about the leftover protected VM that needs a clean up
 * @rc: the RC code of the Destroy Secure Configuration UVC
 * @rrc: the RRC code of the Destroy Secure Configuration UVC
 *
 * Destroy one leftover protected VM.
 * On success, kvm->mm->context.protected_count will be decremented atomically
 * and all other resources used by the VM will be freed.
 *
 * Return: 0 in case of success, otherwise 1
 */
static int kvm_s390_pv_dispose_one_leftover(struct kvm *kvm,
					    struct pv_vm_to_be_destroyed *leftover,
					    u16 *rc, u16 *rrc)
{
	int cc;

	/* It used the destroy-fast UVC, nothing left to do here */
	if (!leftover->handle)
		goto done_fast;
	cc = uv_cmd_nodata(leftover->handle, UVC_CMD_DESTROY_SEC_CONF, rc, rrc);
	KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY LEFTOVER VM: rc %x rrc %x", *rc, *rrc);
	WARN_ONCE(cc, "protvirt destroy leftover vm failed rc %x rrc %x", *rc, *rrc);
	if (cc)
		return cc;
	/*
	 * Intentionally leak unusable memory. If the UVC fails, the memory
	 * used for the VM and its metadata is permanently unusable.
	 * This can only happen in case of a serious KVM or hardware bug; it
	 * is not expected to happen in normal operation.
	 */
	free_pages(leftover->stor_base, get_order(uv_info.guest_base_stor_len));
	free_pages(leftover->old_gmap_table, CRST_ALLOC_ORDER);
	vfree(leftover->stor_var);
done_fast:
	atomic_dec(&kvm->mm->context.protected_count);
	return 0;
}

static int kvm_s390_pv_deinit_vm_fast(struct kvm *kvm, u16 *rc, u16 *rrc)
{
	struct uv_cb_destroy_fast uvcb = {
		.header.cmd = UVC_CMD_DESTROY_SEC_CONF_FAST,
		.header.len = sizeof(uvcb),
		.handle = kvm_s390_pv_get_handle(kvm),
	};
	int cc;

	cc = uv_call_sched(0, (u64)&uvcb);
	if (rc)
		*rc = uvcb.header.rc;
	if (rrc)
		*rrc = uvcb.header.rrc;
	KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM FAST: rc %x rrc %x",
		     uvcb.header.rc, uvcb.header.rrc);
	WARN_ONCE(cc && uvcb.header.rc != 0x104,
		  "protvirt destroy vm fast failed handle %llx rc %x rrc %x",
		  kvm_s390_pv_get_handle(kvm), uvcb.header.rc, uvcb.header.rrc);
	/* Intended memory leak on "impossible" error */
	if (!cc)
		kvm_s390_pv_dealloc_vm(kvm);
	return cc ? -EIO : 0;
}

static inline bool is_destroy_fast_available(void)
{
	return test_bit_inv(BIT_UVC_CMD_DESTROY_SEC_CONF_FAST, uv_info.inst_calls_list);
}

/**
 * kvm_s390_pv_set_aside - Set aside a protected VM for later teardown.
 * @kvm: the VM
 * @rc: return value for the RC field of the UVCB
 * @rrc: return value for the RRC field of the UVCB
 *
 * Set aside the protected VM for a subsequent teardown. The VM will be able
 * to continue immediately as a non-secure VM, and the information needed to
 * properly tear down the protected VM is set aside. If another protected VM
 * was already set aside without starting its teardown, this function will
 * fail.
 * The CPUs of the protected VM need to be destroyed beforehand.
 *
 * Context: kvm->lock needs to be held
 *
 * Return: 0 in case of success, -EINVAL if another protected VM was already set
 * aside, -ENOMEM if the system ran out of memory.
 */
int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc)
{
	struct pv_vm_to_be_destroyed *priv;
	int res = 0;

	lockdep_assert_held(&kvm->lock);
	/*
	 * If another protected VM was already prepared for teardown, refuse.
	 * A normal deinitialization has to be performed instead.
	 */
	if (kvm->arch.pv.set_aside)
		return -EINVAL;

	/* Guest with segment type ASCE, refuse to destroy asynchronously */
	if (kvm->arch.gmap->asce.dt == TABLE_TYPE_SEGMENT)
		return -EINVAL;

	priv = kzalloc_obj(*priv);
	if (!priv)
		return -ENOMEM;

	if (is_destroy_fast_available()) {
		res = kvm_s390_pv_deinit_vm_fast(kvm, rc, rrc);
	} else {
		priv->stor_var = kvm->arch.pv.stor_var;
		priv->stor_base = kvm->arch.pv.stor_base;
		priv->handle = kvm_s390_pv_get_handle(kvm);
		priv->old_gmap_table = (unsigned long)dereference_asce(kvm->arch.gmap->asce);
		if (s390_replace_asce(kvm->arch.gmap))
			res = -ENOMEM;
	}

	if (res) {
		kfree(priv);
		return res;
	}

	gmap_pv_destroy_range(kvm->arch.gmap, 0, gpa_to_gfn(SZ_2G), false);
	kvm_s390_clear_pv_state(kvm);
	kvm->arch.pv.set_aside = priv;

	*rc = UVC_RC_EXECUTED;
	*rrc = 42;
	return 0;
}

/**
 * kvm_s390_pv_deinit_vm - Deinitialize the current protected VM
 * @kvm: the KVM whose protected VM needs to be deinitialized
 * @rc: the RC code of the UVC
 * @rrc: the RRC code of the UVC
 *
 * Deinitialize the current protected VM. This function will destroy and
 * cleanup the current protected VM, but it will not cleanup the guest
 * memory. This function should only be called when the protected VM has
 * just been created and therefore does not have any guest memory, or when
 * the caller cleans up the guest memory separately.
 *
 * This function should not fail, but if it does, the donated memory must
 * not be freed.
 *
 * Context: kvm->lock needs to be held
 *
 * Return: 0 in case of success, otherwise -EIO
 */
int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
{
	int cc;

	cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
			   UVC_CMD_DESTROY_SEC_CONF, rc, rrc);
	if (!cc) {
		atomic_dec(&kvm->mm->context.protected_count);
		kvm_s390_pv_dealloc_vm(kvm);
	} else {
		/* Intended memory leak on "impossible" error */
		s390_replace_asce(kvm->arch.gmap);
	}
	KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY VM: rc %x rrc %x", *rc, *rrc);
	WARN_ONCE(cc, "protvirt destroy vm failed rc %x rrc %x", *rc, *rrc);

	return cc ? -EIO : 0;
}

/**
 * kvm_s390_pv_deinit_cleanup_all - Clean up all protected VMs associated
 * with a specific KVM.
 * @kvm: the KVM to be cleaned up
 * @rc: the RC code of the first failing UVC
 * @rrc: the RRC code of the first failing UVC
 *
 * This function will clean up all protected VMs associated with a KVM.
 * This includes the active one, the one prepared for deinitialization with
 * kvm_s390_pv_set_aside, and any still pending in the need_cleanup list.
 *
 * Context: kvm->lock needs to be held unless being called from
 * kvm_arch_destroy_vm.
 *
 * Return: 0 if all VMs are successfully cleaned up, otherwise -EIO
 */
int kvm_s390_pv_deinit_cleanup_all(struct kvm *kvm, u16 *rc, u16 *rrc)
{
	struct pv_vm_to_be_destroyed *cur;
	bool need_zap = false;
	u16 _rc, _rrc;
	int cc = 0;

	/*
	 * Nothing to do if the counter was already 0. Otherwise make sure
	 * the counter does not reach 0 before calling s390_uv_destroy_range.
	 */
	if (!atomic_inc_not_zero(&kvm->mm->context.protected_count))
		return 0;

	*rc = 1;
	/* If the current VM is protected, destroy it */
	if (kvm_s390_pv_get_handle(kvm)) {
		cc = kvm_s390_pv_deinit_vm(kvm, rc, rrc);
		need_zap = true;
	}

	/* If a previous protected VM was set aside, put it in the need_cleanup list */
	if (kvm->arch.pv.set_aside) {
		list_add(kvm->arch.pv.set_aside, &kvm->arch.pv.need_cleanup);
		kvm->arch.pv.set_aside = NULL;
	}

	/* Cleanup all protected VMs in the need_cleanup list */
	while (!list_empty(&kvm->arch.pv.need_cleanup)) {
		cur = list_first_entry(&kvm->arch.pv.need_cleanup, typeof(*cur), list);
		need_zap = true;
		if (kvm_s390_pv_dispose_one_leftover(kvm, cur, &_rc, &_rrc)) {
			cc = 1;
			/*
			 * Only return the first error rc and rrc, so make
			 * sure it is not overwritten. All destroys will
			 * additionally be reported via KVM_UV_EVENT().
			 */
			if (*rc == UVC_RC_EXECUTED) {
				*rc = _rc;
				*rrc = _rrc;
			}
		}
		list_del(&cur->list);
		kfree(cur);
	}

	/*
	 * If the mm still has a mapping, try to mark all its pages as
	 * accessible. The counter should not reach zero before this
	 * cleanup has been performed.
	 */
	if (need_zap && mmget_not_zero(kvm->mm)) {
		gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), false);
		mmput(kvm->mm);
	}

	/* Now the counter can safely reach 0 */
	atomic_dec(&kvm->mm->context.protected_count);
	return cc ? -EIO : 0;
}

/**
 * kvm_s390_pv_deinit_aside_vm - Teardown a previously set aside protected VM.
 * @kvm: the VM previously associated with the protected VM
 * @rc: return value for the RC field of the UVCB
 * @rrc: return value for the RRC field of the UVCB
 *
 * Tear down the protected VM that had been previously prepared for teardown
 * using kvm_s390_pv_set_aside_vm. Ideally this should be called by
 * userspace asynchronously from a separate thread.
 *
 * Context: kvm->lock must not be held.
 *
 * Return: 0 in case of success, -EINVAL if no protected VM had been
 * prepared for asynchronous teardowm, -EIO in case of other errors.
 */
int kvm_s390_pv_deinit_aside_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
{
	struct pv_vm_to_be_destroyed *p;
	int ret = 0;

	lockdep_assert_not_held(&kvm->lock);
	mutex_lock(&kvm->lock);
	p = kvm->arch.pv.set_aside;
	kvm->arch.pv.set_aside = NULL;
	mutex_unlock(&kvm->lock);
	if (!p)
		return -EINVAL;

	/* When a fatal signal is received, stop immediately */
	if (gmap_pv_destroy_range(kvm->arch.gmap, 0, asce_end(kvm->arch.gmap->asce), true))
		goto done;
	if (kvm_s390_pv_dispose_one_leftover(kvm, p, rc, rrc))
		ret = -EIO;
	kfree(p);
	p = NULL;
done:
	/*
	 * p is not NULL if we aborted because of a fatal signal, in which
	 * case queue the leftover for later cleanup.
	 */
	if (p) {
		mutex_lock(&kvm->lock);
		list_add(&p->list, &kvm->arch.pv.need_cleanup);
		mutex_unlock(&kvm->lock);
		/* Did not finish, but pretend things went well */
		*rc = UVC_RC_EXECUTED;
		*rrc = 42;
	}
	return ret;
}

static void kvm_s390_pv_mmu_notifier_release(struct mmu_notifier *subscription,
					     struct mm_struct *mm)
{
	struct kvm *kvm = container_of(subscription, struct kvm, arch.pv.mmu_notifier);
	u16 dummy;
	int r;

	/*
	 * No locking is needed since this is the last thread of the last user of this
	 * struct mm.
	 * When the struct kvm gets deinitialized, this notifier is also
	 * unregistered. This means that if this notifier runs, then the
	 * struct kvm is still valid.
	 */
	r = kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
	if (!r && is_destroy_fast_available() && kvm_s390_pv_get_handle(kvm))
		kvm_s390_pv_deinit_vm_fast(kvm, &dummy, &dummy);
	set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
}

static const struct mmu_notifier_ops kvm_s390_pv_mmu_notifier_ops = {
	.release = kvm_s390_pv_mmu_notifier_release,
};

int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
{
	struct uv_cb_cgc uvcb = {
		.header.cmd = UVC_CMD_CREATE_SEC_CONF,
		.header.len = sizeof(uvcb)
	};
	int cc, ret;
	u16 dummy;

	/* Add the notifier only once. No races because we hold kvm->lock */
	if (kvm->arch.pv.mmu_notifier.ops != &kvm_s390_pv_mmu_notifier_ops) {
		/* The notifier will be unregistered when the VM is destroyed */
		kvm->arch.pv.mmu_notifier.ops = &kvm_s390_pv_mmu_notifier_ops;
		ret = mmu_notifier_register(&kvm->arch.pv.mmu_notifier, kvm->mm);
		if (ret) {
			kvm->arch.pv.mmu_notifier.ops = NULL;
			return ret;
		}
	}

	ret = kvm_s390_pv_alloc_vm(kvm);
	if (ret)
		return ret;

	/* Inputs */
	uvcb.guest_stor_origin = 0; /* MSO is 0 for KVM */
	uvcb.guest_stor_len = kvm->arch.pv.guest_len;
	uvcb.guest_asce = kvm->arch.gmap->asce.val;
	uvcb.guest_sca = virt_to_phys(kvm->arch.sca);
	uvcb.conf_base_stor_origin =
		virt_to_phys((void *)kvm->arch.pv.stor_base);
	uvcb.conf_virt_stor_origin = (u64)kvm->arch.pv.stor_var;
	uvcb.flags.ap_allow_instr = kvm->arch.model.uv_feat_guest.ap;
	uvcb.flags.ap_instr_intr = kvm->arch.model.uv_feat_guest.ap_intr;

	clear_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
	gmap_split_huge_pages(kvm->arch.gmap);

	cc = uv_call_sched(0, (u64)&uvcb);
	*rc = uvcb.header.rc;
	*rrc = uvcb.header.rrc;
	KVM_UV_EVENT(kvm, 3, "PROTVIRT CREATE VM: handle %llx len %llx rc %x rrc %x flags %04x",
		     uvcb.guest_handle, uvcb.guest_stor_len, *rc, *rrc, uvcb.flags.raw);

	/* Outputs */
	kvm->arch.pv.handle = uvcb.guest_handle;

	atomic_inc(&kvm->mm->context.protected_count);
	if (cc) {
		if (uvcb.header.rc & UVC_RC_NEED_DESTROY) {
			kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
		} else {
			atomic_dec(&kvm->mm->context.protected_count);
			kvm_s390_pv_dealloc_vm(kvm);
		}
		return -EIO;
	}
	return 0;
}

int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc,
			      u16 *rrc)
{
	struct uv_cb_ssc uvcb = {
		.header.cmd = UVC_CMD_SET_SEC_CONF_PARAMS,
		.header.len = sizeof(uvcb),
		.sec_header_origin = (u64)hdr,
		.sec_header_len = length,
		.guest_handle = kvm_s390_pv_get_handle(kvm),
	};
	int cc = uv_call(0, (u64)&uvcb);

	*rc = uvcb.header.rc;
	*rrc = uvcb.header.rrc;
	KVM_UV_EVENT(kvm, 3, "PROTVIRT VM SET PARMS: rc %x rrc %x",
		     *rc, *rrc);
	return cc ? -EINVAL : 0;
}

static int unpack_one(struct kvm *kvm, unsigned long addr, u64 tweak,
		      u64 offset, u16 *rc, u16 *rrc)
{
	struct uv_cb_unp uvcb = {
		.header.cmd = UVC_CMD_UNPACK_IMG,
		.header.len = sizeof(uvcb),
		.guest_handle = kvm_s390_pv_get_handle(kvm),
		.gaddr = addr,
		.tweak[0] = tweak,
		.tweak[1] = offset,
	};
	int ret = kvm_s390_pv_make_secure(kvm, addr, &uvcb);

	*rc = uvcb.header.rc;
	*rrc = uvcb.header.rrc;

	if (ret == -ENXIO) {
		ret = kvm_s390_faultin_gfn_simple(NULL, kvm, gpa_to_gfn(addr), true);
		if (!ret)
			return -EAGAIN;
	}

	if (ret && ret != -EAGAIN)
		KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: failed addr %llx with rc %x rrc %x",
			     uvcb.gaddr, *rc, *rrc);
	return ret;
}

int kvm_s390_pv_unpack(struct kvm *kvm, unsigned long addr, unsigned long size,
		       unsigned long tweak, u16 *rc, u16 *rrc)
{
	u64 offset = 0;
	int ret = 0;

	if (addr & ~PAGE_MASK || !size || size & ~PAGE_MASK)
		return -EINVAL;

	KVM_UV_EVENT(kvm, 3, "PROTVIRT VM UNPACK: start addr %lx size %lx",
		     addr, size);

	guard(srcu)(&kvm->srcu);

	while (offset < size) {
		ret = unpack_one(kvm, addr, tweak, offset, rc, rrc);
		if (ret == -EAGAIN) {
			cond_resched();
			if (fatal_signal_pending(current))
				break;
			continue;
		}
		if (ret)
			break;
		addr += PAGE_SIZE;
		offset += PAGE_SIZE;
	}
	if (!ret)
		KVM_UV_EVENT(kvm, 3, "%s", "PROTVIRT VM UNPACK: successful");
	return ret;
}

int kvm_s390_pv_set_cpu_state(struct kvm_vcpu *vcpu, u8 state)
{
	struct uv_cb_cpu_set_state uvcb = {
		.header.cmd	= UVC_CMD_CPU_SET_STATE,
		.header.len	= sizeof(uvcb),
		.cpu_handle	= kvm_s390_pv_cpu_get_handle(vcpu),
		.state		= state,
	};
	int cc;

	cc = uv_call(0, (u64)&uvcb);
	KVM_UV_EVENT(vcpu->kvm, 3, "PROTVIRT SET CPU %d STATE %d rc %x rrc %x",
		     vcpu->vcpu_id, state, uvcb.header.rc, uvcb.header.rrc);
	if (cc)
		return -EINVAL;
	return 0;
}

int kvm_s390_pv_dump_cpu(struct kvm_vcpu *vcpu, void *buff, u16 *rc, u16 *rrc)
{
	struct uv_cb_dump_cpu uvcb = {
		.header.cmd = UVC_CMD_DUMP_CPU,
		.header.len = sizeof(uvcb),
		.cpu_handle = vcpu->arch.pv.handle,
		.dump_area_origin = (u64)buff,
	};
	int cc;

	cc = uv_call_sched(0, (u64)&uvcb);
	*rc = uvcb.header.rc;
	*rrc = uvcb.header.rrc;
	return cc;
}

/* Size of the cache for the storage state dump data. 1MB for now */
#define DUMP_BUFF_LEN HPAGE_SIZE

/**
 * kvm_s390_pv_dump_stor_state
 *
 * @kvm: pointer to the guest's KVM struct
 * @buff_user: Userspace pointer where we will write the results to
 * @gaddr: Starting absolute guest address for which the storage state
 *	   is requested.
 * @buff_user_len: Length of the buff_user buffer
 * @rc: Pointer to where the uvcb return code is stored
 * @rrc: Pointer to where the uvcb return reason code is stored
 *
 * Stores buff_len bytes of tweak component values to buff_user
 * starting with the 1MB block specified by the absolute guest address
 * (gaddr). The gaddr pointer will be updated with the last address
 * for which data was written when returning to userspace. buff_user
 * might be written to even if an error rc is returned. For instance
 * if we encounter a fault after writing the first page of data.
 *
 * Context: kvm->lock needs to be held
 *
 * Return:
 *  0 on success
 *  -ENOMEM if allocating the cache fails
 *  -EINVAL if gaddr is not aligned to 1MB
 *  -EINVAL if buff_user_len is not aligned to uv_info.conf_dump_storage_state_len
 *  -EINVAL if the UV call fails, rc and rrc will be set in this case
 *  -EFAULT if copying the result to buff_user failed
 */
int kvm_s390_pv_dump_stor_state(struct kvm *kvm, void __user *buff_user,
				u64 *gaddr, u64 buff_user_len, u16 *rc, u16 *rrc)
{
	struct uv_cb_dump_stor_state uvcb = {
		.header.cmd = UVC_CMD_DUMP_CONF_STOR_STATE,
		.header.len = sizeof(uvcb),
		.config_handle = kvm->arch.pv.handle,
		.gaddr = *gaddr,
		.dump_area_origin = 0,
	};
	const u64 increment_len = uv_info.conf_dump_storage_state_len;
	size_t buff_kvm_size;
	size_t size_done = 0;
	u8 *buff_kvm = NULL;
	int cc, ret;

	ret = -EINVAL;
	/* UV call processes 1MB guest storage chunks at a time */
	if (!IS_ALIGNED(*gaddr, HPAGE_SIZE))
		goto out;

	/*
	 * We provide the storage state for 1MB chunks of guest
	 * storage. The buffer will need to be aligned to
	 * conf_dump_storage_state_len so we don't end on a partial
	 * chunk.
	 */
	if (!buff_user_len ||
	    !IS_ALIGNED(buff_user_len, increment_len))
		goto out;

	/*
	 * Allocate a buffer from which we will later copy to the user
	 * process. We don't want userspace to dictate our buffer size
	 * so we limit it to DUMP_BUFF_LEN.
	 */
	ret = -ENOMEM;
	buff_kvm_size = min_t(u64, buff_user_len, DUMP_BUFF_LEN);
	buff_kvm = vzalloc(buff_kvm_size);
	if (!buff_kvm)
		goto out;

	ret = 0;
	uvcb.dump_area_origin = (u64)buff_kvm;
	/* We will loop until the user buffer is filled or an error occurs */
	do {
		/* Get 1MB worth of guest storage state data */
		cc = uv_call_sched(0, (u64)&uvcb);

		/* All or nothing */
		if (cc) {
			ret = -EINVAL;
			break;
		}

		size_done += increment_len;
		uvcb.dump_area_origin += increment_len;
		buff_user_len -= increment_len;
		uvcb.gaddr += HPAGE_SIZE;

		/* KVM Buffer full, time to copy to the process */
		if (!buff_user_len || size_done == DUMP_BUFF_LEN) {
			if (copy_to_user(buff_user, buff_kvm, size_done)) {
				ret = -EFAULT;
				break;
			}

			buff_user += size_done;
			size_done = 0;
			uvcb.dump_area_origin = (u64)buff_kvm;
		}
	} while (buff_user_len);

	/* Report back where we ended dumping */
	*gaddr = uvcb.gaddr;

	/* Lets only log errors, we don't want to spam */
out:
	if (ret)
		KVM_UV_EVENT(kvm, 3,
			     "PROTVIRT DUMP STORAGE STATE: addr %llx ret %d, uvcb rc %x rrc %x",
			     uvcb.gaddr, ret, uvcb.header.rc, uvcb.header.rrc);
	*rc = uvcb.header.rc;
	*rrc = uvcb.header.rrc;
	vfree(buff_kvm);

	return ret;
}

/**
 * kvm_s390_pv_dump_complete
 *
 * @kvm: pointer to the guest's KVM struct
 * @buff_user: Userspace pointer where we will write the results to
 * @rc: Pointer to where the uvcb return code is stored
 * @rrc: Pointer to where the uvcb return reason code is stored
 *
 * Completes the dumping operation and writes the completion data to
 * user space.
 *
 * Context: kvm->lock needs to be held
 *
 * Return:
 *  0 on success
 *  -ENOMEM if allocating the completion buffer fails
 *  -EINVAL if the UV call fails, rc and rrc will be set in this case
 *  -EFAULT if copying the result to buff_user failed
 */
int kvm_s390_pv_dump_complete(struct kvm *kvm, void __user *buff_user,
			      u16 *rc, u16 *rrc)
{
	struct uv_cb_dump_complete complete = {
		.header.len = sizeof(complete),
		.header.cmd = UVC_CMD_DUMP_COMPLETE,
		.config_handle = kvm_s390_pv_get_handle(kvm),
	};
	u64 *compl_data;
	int ret;

	/* Allocate dump area */
	compl_data = vzalloc(uv_info.conf_dump_finalize_len);
	if (!compl_data)
		return -ENOMEM;
	complete.dump_area_origin = (u64)compl_data;

	ret = uv_call_sched(0, (u64)&complete);
	*rc = complete.header.rc;
	*rrc = complete.header.rrc;
	KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP COMPLETE: rc %x rrc %x",
		     complete.header.rc, complete.header.rrc);

	if (!ret) {
		/*
		 * kvm_s390_pv_dealloc_vm() will also (mem)set
		 * this to false on a reboot or other destroy
		 * operation for this vm.
		 */
		kvm->arch.pv.dumping = false;
		kvm_s390_vcpu_unblock_all(kvm);
		ret = copy_to_user(buff_user, compl_data, uv_info.conf_dump_finalize_len);
		if (ret)
			ret = -EFAULT;
	}
	vfree(compl_data);
	/* If the UVC returned an error, translate it to -EINVAL */
	if (ret > 0)
		ret = -EINVAL;
	return ret;
}