/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */
/* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */

/*
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2022 Oxide Computer Company
 */

#include <sys/cpuvar.h>
#include <sys/debug.h>
#include <sys/kmem.h>
#include <sys/ksynch.h>
#include <sys/list.h>
#include <sys/types.h>
#include <sys/vmm.h>
#include <sys/vmm_kernel.h>
#include <sys/vmm_impl.h>
#include <sys/zone.h>

/*
 * zone specific data
 *
 * Zone specific data is used to keep an association between zones and the vmm
 * instances that may be running in them.  This is used to ensure that vmm
 * instances do not outlive their parent zone.
 *
 * Locking strategy
 *
 * The global vmm_zsd_lock is held while modifying vmm_zsd_list.
 *
 * The per zone vz_lock in vmm_zsd_t is held while reading or writing anything
 * within in vmm_zsd_t instance.  This is important to ensure that there's not
 * an accidental VM creating as a zone is going down.
 */

/*
 * One of these per zone.
 */
struct vmm_zsd {
	list_t		vz_vmms;	/* vmm instances in the zone */
	list_node_t	vz_linkage;	/* link to other zones */
	boolean_t	vz_active;	/* B_FALSE early in shutdown callback */
	zoneid_t	vz_zoneid;
	kmutex_t	vz_lock;
};

static kmutex_t vmm_zsd_lock;		/* Protects vmm_zsd_list */
static list_t vmm_zsd_list;		/* Linkage between all zsd instances */

static zone_key_t vmm_zsd_key;

int
vmm_zsd_add_vm(vmm_softc_t *sc)
{
	vmm_zsd_t *zsd;

	ASSERT(sc->vmm_zone != NULL);

	mutex_enter(&vmm_zsd_lock);

	for (zsd = list_head(&vmm_zsd_list); zsd != NULL;
	    zsd = list_next(&vmm_zsd_list, zsd)) {
		if (zsd->vz_zoneid == sc->vmm_zone->zone_id) {
			break;
		}
	}

	VERIFY(zsd != NULL);
	mutex_exit(&vmm_zsd_lock);

	mutex_enter(&zsd->vz_lock);
	if (!zsd->vz_active) {
		mutex_exit(&zsd->vz_lock);
		return (ENOSYS);
	}

	sc->vmm_zsd = zsd;
	list_insert_tail(&zsd->vz_vmms, sc);

	mutex_exit(&zsd->vz_lock);

	return (0);
}

void
vmm_zsd_rem_vm(vmm_softc_t *sc)
{
	vmm_zsd_t *zsd = sc->vmm_zsd;

	mutex_enter(&zsd->vz_lock);

	list_remove(&zsd->vz_vmms, sc);
	sc->vmm_zsd = NULL;

	mutex_exit(&zsd->vz_lock);
}

static void *
vmm_zsd_create(zoneid_t zid)
{
	vmm_zsd_t *zsd;
	zone_t *zone;

	zsd = kmem_zalloc(sizeof (*zsd), KM_SLEEP);

	list_create(&zsd->vz_vmms, sizeof (vmm_softc_t),
	    offsetof(vmm_softc_t, vmm_zsd_linkage));

	zsd->vz_zoneid = zid;

	mutex_init(&zsd->vz_lock, NULL, MUTEX_DEFAULT, NULL);

	/*
	 * If the vmm module is loaded while this zone is in the midst of
	 * shutting down, vmm_zsd_destroy() may be called without
	 * vmm_zsd_shutdown() ever being called. If it is shutting down, there
	 * is no sense in letting any in-flight VM creation succeed so set
	 * vz_active accordingly.
	 *
	 * zone_find_by_id_nolock() is used rather than zone_find_by_id()
	 * so that the zone is returned regardless of state.
	 */
	zone = zone_find_by_id_nolock(zid);
	VERIFY(zone != NULL);
	zsd->vz_active = zone_status_get(zone) < ZONE_IS_SHUTTING_DOWN;

	mutex_enter(&vmm_zsd_lock);
	list_insert_tail(&vmm_zsd_list, zsd);
	mutex_exit(&vmm_zsd_lock);

	return (zsd);
}

/*
 * Tells all running VMs in the zone to poweroff.  This does not reclaim guest
 * resources (memory, etc.).
 */
static void
vmm_zsd_shutdown(zoneid_t zid, void *data)
{
	vmm_zsd_t *zsd = data;
	vmm_softc_t *sc;

	mutex_enter(&zsd->vz_lock);

	/*
	 * This may already be B_FALSE. See comment in vmm_zsd_create(). If it
	 * is already B_FALSE we will take a quick trip through the empty list.
	 */
	zsd->vz_active = B_FALSE;

	for (sc = list_head(&zsd->vz_vmms); sc != NULL;
	    sc = list_next(&zsd->vz_vmms, sc)) {
		/* Send a poweroff to the VM, whether running or not. */
		(void) vm_suspend(sc->vmm_vm, VM_SUSPEND_POWEROFF);
	}
	mutex_exit(&zsd->vz_lock);
}

/*
 * Reap all VMs that remain and free up guest resources.
 */
static void
vmm_zsd_destroy(zoneid_t zid, void *data)
{
	vmm_zsd_t *zsd = data;
	vmm_softc_t *sc;

	mutex_enter(&vmm_zsd_lock);
	list_remove(&vmm_zsd_list, zsd);
	mutex_exit(&vmm_zsd_lock);

	mutex_enter(&zsd->vz_lock);
	ASSERT(!zsd->vz_active);

	while ((sc = list_remove_head(&zsd->vz_vmms)) != NULL) {
		vmm_zone_vm_destroy(sc);
	}

	mutex_exit(&zsd->vz_lock);
	mutex_destroy(&zsd->vz_lock);

	kmem_free(zsd, sizeof (*zsd));
}

void
vmm_zsd_init(void)
{
	mutex_init(&vmm_zsd_lock, NULL, MUTEX_DEFAULT, NULL);
	list_create(&vmm_zsd_list, sizeof (vmm_zsd_t),
	    offsetof(vmm_zsd_t, vz_linkage));
	zone_key_create(&vmm_zsd_key, vmm_zsd_create, vmm_zsd_shutdown,
	    vmm_zsd_destroy);
}

void
vmm_zsd_fini(void)
{
	/* Calls vmm_zsd_destroy() on all zones. */
	VERIFY0(zone_key_delete(vmm_zsd_key));

	ASSERT(list_is_empty(&vmm_zsd_list));
	list_destroy(&vmm_zsd_list);
	mutex_destroy(&vmm_zsd_lock);
}