xref: /freebsd/sys/dev/vmware/vmci/vmci.c (revision fdafd315ad0d0f28a11b9fb4476a9ab059c62b92)

/*-
 * Copyright (c) 2018 VMware, Inc.
 *
 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
 */

/* Driver for VMware Virtual Machine Communication Interface (VMCI) device. */

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/rman.h>
#include <sys/systm.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#include <machine/bus.h>

#include "vmci.h"
#include "vmci_doorbell.h"
#include "vmci_driver.h"
#include "vmci_kernel_defs.h"
#include "vmci_queue_pair.h"

static int	vmci_probe(device_t);
static int	vmci_attach(device_t);
static int	vmci_detach(device_t);
static int	vmci_shutdown(device_t);

static int	vmci_map_bars(struct vmci_softc *);
static void	vmci_unmap_bars(struct vmci_softc *);

static int	vmci_config_capabilities(struct vmci_softc *);

static int	vmci_dma_malloc_int(struct vmci_softc *, bus_size_t,
		    bus_size_t, struct vmci_dma_alloc *);
static void	vmci_dma_free_int(struct vmci_softc *,
		    struct vmci_dma_alloc *);

static int	vmci_config_interrupts(struct vmci_softc *);
static int	vmci_config_interrupt(struct vmci_softc *);
static int	vmci_check_intr_cnt(struct vmci_softc *);
static int	vmci_allocate_interrupt_resources(struct vmci_softc *);
static int	vmci_setup_interrupts(struct vmci_softc *);
static void	vmci_dismantle_interrupts(struct vmci_softc *);
static void	vmci_interrupt(void *);
static void	vmci_interrupt_bm(void *);
static void	dispatch_datagrams(void *, int);
static void	process_bitmap(void *, int);

static void	vmci_delayed_work_fn_cb(void *context, int data);

static device_method_t vmci_methods[] = {
	/* Device interface. */
	DEVMETHOD(device_probe,		vmci_probe),
	DEVMETHOD(device_attach,	vmci_attach),
	DEVMETHOD(device_detach,	vmci_detach),
	DEVMETHOD(device_shutdown,	vmci_shutdown),

	DEVMETHOD_END
};

static driver_t vmci_driver = {
	"vmci", vmci_methods, sizeof(struct vmci_softc)
};

DRIVER_MODULE(vmci, pci, vmci_driver, 0, 0);
MODULE_VERSION(vmci, VMCI_VERSION);
const struct {
	uint16_t vendor;
	uint16_t device;
	const char *desc;
} vmci_ids[] = {
	{ VMCI_VMWARE_VENDOR_ID, VMCI_VMWARE_DEVICE_ID,
	    "VMware Virtual Machine Communication Interface" },
};
MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, vmci, vmci_ids,
    nitems(vmci_ids));

MODULE_DEPEND(vmci, pci, 1, 1, 1);

static struct vmci_softc *vmci_sc;

#define LGPFX	"vmci: "
/*
 * Allocate a buffer for incoming datagrams globally to avoid repeated
 * allocation in the interrupt handler's atomic context.
 */
static uint8_t *data_buffer = NULL;
static uint32_t data_buffer_size = VMCI_MAX_DG_SIZE;

struct vmci_delayed_work_info {
	vmci_work_fn	*work_fn;
	void		*data;
	vmci_list_item(vmci_delayed_work_info) entry;
};

/*
 *------------------------------------------------------------------------------
 *
 * vmci_probe --
 *
 *     Probe to see if the VMCI device is present.
 *
 * Results:
 *     BUS_PROBE_DEFAULT if device exists, ENXIO otherwise.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_probe(device_t dev)
{

	if (pci_get_vendor(dev) == vmci_ids[0].vendor &&
	    pci_get_device(dev) == vmci_ids[0].device) {
		device_set_desc(dev, vmci_ids[0].desc);

		return (BUS_PROBE_DEFAULT);
	}

	return (ENXIO);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_attach --
 *
 *     Attach VMCI device to the system after vmci_probe() has been called and
 *     the device has been detected.
 *
 * Results:
 *     0 if success, ENXIO otherwise.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_attach(device_t dev)
{
	struct vmci_softc *sc;
	int error, i;

	sc = device_get_softc(dev);
	sc->vmci_dev = dev;
	vmci_sc = sc;

	data_buffer = NULL;
	sc->vmci_num_intr = 0;
	for (i = 0; i < VMCI_MAX_INTRS; i++) {
		sc->vmci_intrs[i].vmci_irq = NULL;
		sc->vmci_intrs[i].vmci_handler = NULL;
	}

	TASK_INIT(&sc->vmci_interrupt_dq_task, 0, dispatch_datagrams, sc);
	TASK_INIT(&sc->vmci_interrupt_bm_task, 0, process_bitmap, sc);

	TASK_INIT(&sc->vmci_delayed_work_task, 0, vmci_delayed_work_fn_cb, sc);

	pci_enable_busmaster(dev);

	mtx_init(&sc->vmci_spinlock, "VMCI Spinlock", NULL, MTX_SPIN);
	mtx_init(&sc->vmci_delayed_work_lock, "VMCI Delayed Work Lock",
	    NULL, MTX_DEF);

	error = vmci_map_bars(sc);
	if (error) {
		VMCI_LOG_ERROR(LGPFX"Failed to map PCI BARs.\n");
		goto fail;
	}

	error = vmci_config_capabilities(sc);
	if (error) {
		VMCI_LOG_ERROR(LGPFX"Failed to configure capabilities.\n");
		goto fail;
	}

	vmci_list_init(&sc->vmci_delayed_work_infos);

	vmci_components_init();
	vmci_util_init();
	error = vmci_qp_guest_endpoints_init();
	if (error) {
		VMCI_LOG_ERROR(LGPFX"vmci_qp_guest_endpoints_init failed.\n");
		goto fail;
	}

	error = vmci_config_interrupts(sc);
	if (error)
		VMCI_LOG_ERROR(LGPFX"Failed to enable interrupts.\n");

fail:
	if (error) {
		vmci_detach(dev);
		return (ENXIO);
	}

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_detach --
 *
 *     Detach the VMCI device.
 *
 * Results:
 *     0
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_detach(device_t dev)
{
	struct vmci_softc *sc;

	sc = device_get_softc(dev);

	vmci_qp_guest_endpoints_exit();
	vmci_util_exit();

	vmci_dismantle_interrupts(sc);

	vmci_components_cleanup();

	if mtx_initialized(&sc->vmci_spinlock) {
		taskqueue_drain(taskqueue_thread, &sc->vmci_delayed_work_task);
		mtx_destroy(&sc->vmci_delayed_work_lock);
	}

	if (sc->vmci_res0 != NULL)
		bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
		    VMCI_CONTROL_ADDR, VMCI_CONTROL_RESET);

	if (sc->vmci_notifications_bitmap.dma_vaddr != NULL)
		vmci_dma_free(&sc->vmci_notifications_bitmap);

	vmci_unmap_bars(sc);

	if mtx_initialized(&sc->vmci_spinlock)
		mtx_destroy(&sc->vmci_spinlock);

	pci_disable_busmaster(dev);

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_shutdown --
 *
 *     This function is called during system shutdown. We don't do anything.
 *
 * Results:
 *     0
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_shutdown(device_t dev)
{

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_map_bars --
 *
 *     Maps the PCI I/O and MMIO BARs.
 *
 * Results:
 *     0 on success, ENXIO otherwise.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_map_bars(struct vmci_softc *sc)
{
	int rid;

	/* Map the PCI I/O BAR: BAR0 */
	rid = PCIR_BAR(0);
	sc->vmci_res0 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IOPORT,
	    &rid, RF_ACTIVE);
	if (sc->vmci_res0 == NULL) {
		VMCI_LOG_ERROR(LGPFX"Could not map: BAR0\n");
		return (ENXIO);
	}

	sc->vmci_iot0 = rman_get_bustag(sc->vmci_res0);
	sc->vmci_ioh0 = rman_get_bushandle(sc->vmci_res0);
	sc->vmci_ioaddr = rman_get_start(sc->vmci_res0);

	/* Map the PCI MMIO BAR: BAR1 */
	rid = PCIR_BAR(1);
	sc->vmci_res1 = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_MEMORY,
	    &rid, RF_ACTIVE);
	if (sc->vmci_res1 == NULL) {
		VMCI_LOG_ERROR(LGPFX"Could not map: BAR1\n");
		return (ENXIO);
	}

	sc->vmci_iot1 = rman_get_bustag(sc->vmci_res1);
	sc->vmci_ioh1 = rman_get_bushandle(sc->vmci_res1);

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_unmap_bars --
 *
 *     Unmaps the VMCI PCI I/O and MMIO BARs.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_unmap_bars(struct vmci_softc *sc)
{
	int rid;

	if (sc->vmci_res0 != NULL) {
		rid = PCIR_BAR(0);
		bus_release_resource(sc->vmci_dev, SYS_RES_IOPORT, rid,
		    sc->vmci_res0);
		sc->vmci_res0 = NULL;
	}

	if (sc->vmci_res1 != NULL) {
		rid = PCIR_BAR(1);
		bus_release_resource(sc->vmci_dev, SYS_RES_MEMORY, rid,
		    sc->vmci_res1);
		sc->vmci_res1 = NULL;
	}
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_config_capabilities --
 *
 *     Check the VMCI device capabilities and configure the device accordingly.
 *
 * Results:
 *     0 if success, ENODEV otherwise.
 *
 * Side effects:
 *     Device capabilities are enabled.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_config_capabilities(struct vmci_softc *sc)
{
	unsigned long bitmap_PPN;
	int error;

	/*
	 * Verify that the VMCI device supports the capabilities that we
	 * need. Datagrams are necessary and notifications will be used
	 * if the device supports it.
	 */
	sc->capabilities = bus_space_read_4(sc->vmci_iot0, sc->vmci_ioh0,
	    VMCI_CAPS_ADDR);

	if ((sc->capabilities & VMCI_CAPS_DATAGRAM) == 0) {
		VMCI_LOG_ERROR(LGPFX"VMCI device does not support "
		    "datagrams.\n");
		return (ENODEV);
	}

	if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
		sc->capabilities = VMCI_CAPS_DATAGRAM;
		error = vmci_dma_malloc(PAGE_SIZE, 1,
		    &sc->vmci_notifications_bitmap);
		if (error)
			VMCI_LOG_ERROR(LGPFX"Failed to alloc memory for "
			    "notification bitmap.\n");
		else {
			memset(sc->vmci_notifications_bitmap.dma_vaddr, 0,
			    PAGE_SIZE);
			sc->capabilities |= VMCI_CAPS_NOTIFICATIONS;
		}
	} else
		sc->capabilities = VMCI_CAPS_DATAGRAM;

	/* Let the host know which capabilities we intend to use. */
	bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
	    VMCI_CAPS_ADDR, sc->capabilities);

	/*
	 * Register notification bitmap with device if that capability is
	 * used.
	 */
	if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS) {
		bitmap_PPN =
		    sc->vmci_notifications_bitmap.dma_paddr >> PAGE_SHIFT;
		vmci_register_notification_bitmap(bitmap_PPN);
	}

	/* Check host capabilities. */
	if (!vmci_check_host_capabilities())
		return (ENODEV);

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dmamap_cb --
 *
 *     Callback to receive mapping information resulting from the load of a
 *     bus_dmamap_t via bus_dmamap_load()
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
{
	bus_addr_t *baddr = arg;

	if (error == 0)
		*baddr = segs->ds_addr;
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dma_malloc_int --
 *
 *     Internal function that allocates DMA memory.
 *
 * Results:
 *     0 if success.
 *     ENOMEM if insufficient memory.
 *     EINPROGRESS if mapping is deferred.
 *     EINVAL if the request was invalid.
 *
 * Side effects:
 *     DMA memory is allocated.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_dma_malloc_int(struct vmci_softc *sc, bus_size_t size, bus_size_t align,
    struct vmci_dma_alloc *dma)
{
	int error;

	bzero(dma, sizeof(struct vmci_dma_alloc));

	error = bus_dma_tag_create(bus_get_dma_tag(vmci_sc->vmci_dev),
	    align, 0,		/* alignment, bounds */
	    BUS_SPACE_MAXADDR,	/* lowaddr */
	    BUS_SPACE_MAXADDR,	/* highaddr */
	    NULL, NULL,		/* filter, filterarg */
	    size,		/* maxsize */
	    1,			/* nsegments */
	    size,		/* maxsegsize */
	    BUS_DMA_ALLOCNOW,	/* flags */
	    NULL,		/* lockfunc */
	    NULL,		/* lockfuncarg */
	    &dma->dma_tag);
	if (error) {
		VMCI_LOG_ERROR(LGPFX"bus_dma_tag_create failed: %d\n", error);
		goto fail;
	}

	error = bus_dmamem_alloc(dma->dma_tag, (void **)&dma->dma_vaddr,
	    BUS_DMA_ZERO | BUS_DMA_NOWAIT, &dma->dma_map);
	if (error) {
		VMCI_LOG_ERROR(LGPFX"bus_dmamem_alloc failed: %d\n", error);
		goto fail;
	}

	error = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
	    size, vmci_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
	if (error) {
		VMCI_LOG_ERROR(LGPFX"bus_dmamap_load failed: %d\n", error);
		goto fail;
	}

	dma->dma_size = size;

fail:
	if (error)
		vmci_dma_free(dma);

	return (error);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dma_malloc --
 *
 *     This function is a wrapper around vmci_dma_malloc_int for callers
 *     outside of this module. Since we only support a single VMCI device, this
 *     wrapper provides access to the device softc structure.
 *
 * Results:
 *     0 if success.
 *     ENOMEM if insufficient memory.
 *     EINPROGRESS if mapping is deferred.
 *     EINVAL if the request was invalid.
 *
 * Side effects:
 *     DMA memory is allocated.
 *
 *------------------------------------------------------------------------------
 */

int
vmci_dma_malloc(bus_size_t size, bus_size_t align, struct vmci_dma_alloc *dma)
{

	return (vmci_dma_malloc_int(vmci_sc, size, align, dma));
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dma_free_int --
 *
 *     Internal function that frees DMA memory.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     Frees DMA memory.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_dma_free_int(struct vmci_softc *sc, struct vmci_dma_alloc *dma)
{

	if (dma->dma_tag != NULL) {
		if (dma->dma_paddr != 0) {
			bus_dmamap_sync(dma->dma_tag, dma->dma_map,
			    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
			bus_dmamap_unload(dma->dma_tag, dma->dma_map);
		}

		if (dma->dma_vaddr != NULL)
			bus_dmamem_free(dma->dma_tag, dma->dma_vaddr,
			    dma->dma_map);

		bus_dma_tag_destroy(dma->dma_tag);
	}
	bzero(dma, sizeof(struct vmci_dma_alloc));
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dma_free --
 *
 *     This function is a wrapper around vmci_dma_free_int for callers outside
 *     of this module. Since we only support a single VMCI device, this wrapper
 *     provides access to the device softc structure.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     Frees DMA memory.
 *
 *------------------------------------------------------------------------------
 */

void
vmci_dma_free(struct vmci_dma_alloc *dma)
{

	vmci_dma_free_int(vmci_sc, dma);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_config_interrupts --
 *
 *     Configures and enables interrupts. Try to configure MSI-X. If this fails,
 *     try to configure MSI. If even this fails, try legacy interrupts.
 *
 * Results:
 *     0 if success.
 *     ENOMEM if insufficient memory.
 *     ENODEV if the device doesn't support interrupts.
 *     ENXIO if the device configuration failed.
 *
 * Side effects:
 *     Interrupts get enabled if successful.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_config_interrupts(struct vmci_softc *sc)
{
	int error;

	data_buffer = malloc(data_buffer_size, M_DEVBUF, M_ZERO | M_NOWAIT);
	if (data_buffer == NULL)
		return (ENOMEM);

	sc->vmci_intr_type = VMCI_INTR_TYPE_MSIX;
	error = vmci_config_interrupt(sc);
	if (error) {
		sc->vmci_intr_type = VMCI_INTR_TYPE_MSI;
		error = vmci_config_interrupt(sc);
	}
	if (error) {
		sc->vmci_intr_type = VMCI_INTR_TYPE_INTX;
		error = vmci_config_interrupt(sc);
	}
	if (error)
		return (error);

	/* Enable specific interrupt bits. */
	if (sc->capabilities & VMCI_CAPS_NOTIFICATIONS)
		bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
		    VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM | VMCI_IMR_NOTIFICATION);
	else
		bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
		    VMCI_IMR_ADDR, VMCI_IMR_DATAGRAM);

	/* Enable interrupts. */
	bus_space_write_4(sc->vmci_iot0, sc->vmci_ioh0,
	    VMCI_CONTROL_ADDR, VMCI_CONTROL_INT_ENABLE);

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_config_interrupt --
 *
 *     Check the number of interrupts supported, allocate resources and setup
 *     interrupts.
 *
 * Results:
 *     0 if success.
 *     ENOMEM if insufficient memory.
 *     ENODEV if the device doesn't support interrupts.
 *     ENXIO if the device configuration failed.
 *
 * Side effects:
 *     Resources get allocated and interrupts get setup (but not enabled) if
 *     successful.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_config_interrupt(struct vmci_softc *sc)
{
	int error;

	error = vmci_check_intr_cnt(sc);
	if (error)
		return (error);

	error = vmci_allocate_interrupt_resources(sc);
	if (error)
		return (error);

	error = vmci_setup_interrupts(sc);
	if (error)
		return (error);

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_check_intr_cnt --
 *
 *     Check the number of interrupts supported by the device and ask PCI bus
 *     to allocate appropriate number of interrupts.
 *
 * Results:
 *     0 if success.
 *     ENODEV if the device doesn't support any interrupts.
 *     ENXIO if the device configuration failed.
 *
 * Side effects:
 *     Resources get allocated on success.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_check_intr_cnt(struct vmci_softc *sc)
{

	if (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) {
		sc->vmci_num_intr = 1;
		return (0);
	}

	/*
	 * Make sure that the device supports the required number of MSI/MSI-X
	 * messages. We try for 2 MSI-X messages but 1 is good too. We need at
	 * least 1 MSI message.
	 */
	sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
	    pci_msix_count(sc->vmci_dev) : pci_msi_count(sc->vmci_dev);

	if (!sc->vmci_num_intr) {
		VMCI_LOG_ERROR(LGPFX"Device does not support any interrupt"
		    " messages");
		return (ENODEV);
	}

	sc->vmci_num_intr = (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) ?
	    VMCI_MAX_INTRS : 1;
	if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSIX) {
		if (pci_alloc_msix(sc->vmci_dev, &sc->vmci_num_intr))
			return (ENXIO);
	} else if (sc->vmci_intr_type == VMCI_INTR_TYPE_MSI) {
		if (pci_alloc_msi(sc->vmci_dev, &sc->vmci_num_intr))
			return (ENXIO);
	}

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_allocate_interrupt_resources --
 *
 *     Allocate resources necessary for interrupts.
 *
 * Results:
 *     0 if success, ENXIO otherwise.
 *
 * Side effects:
 *     Resources get allocated on success.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_allocate_interrupt_resources(struct vmci_softc *sc)
{
	struct resource *irq;
	int flags, i, rid;

	flags = RF_ACTIVE;
	flags |= (sc->vmci_num_intr == 1) ? RF_SHAREABLE : 0;
	rid = (sc->vmci_intr_type == VMCI_INTR_TYPE_INTX) ? 0 : 1;

	for (i = 0; i < sc->vmci_num_intr; i++, rid++) {
		irq = bus_alloc_resource_any(sc->vmci_dev, SYS_RES_IRQ, &rid,
		    flags);
		if (irq == NULL)
			return (ENXIO);
		sc->vmci_intrs[i].vmci_irq = irq;
		sc->vmci_intrs[i].vmci_rid = rid;
	}

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_setup_interrupts --
 *
 *     Sets up the interrupts.
 *
 * Results:
 *     0 if success, appropriate error code from bus_setup_intr otherwise.
 *
 * Side effects:
 *     Interrupt handler gets attached.
 *
 *------------------------------------------------------------------------------
 */

static int
vmci_setup_interrupts(struct vmci_softc *sc)
{
	struct vmci_interrupt *intr;
	int error, flags;

	flags = INTR_TYPE_NET | INTR_MPSAFE;
	if (sc->vmci_num_intr > 1)
		flags |= INTR_EXCL;

	intr = &sc->vmci_intrs[0];
	error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags, NULL,
	    vmci_interrupt, NULL, &intr->vmci_handler);
	if (error)
		return (error);

	if (sc->vmci_num_intr == 2) {
		bus_describe_intr(sc->vmci_dev, intr->vmci_irq,
		    intr->vmci_handler, "dg");
		intr = &sc->vmci_intrs[1];
		error = bus_setup_intr(sc->vmci_dev, intr->vmci_irq, flags,
		    NULL, vmci_interrupt_bm, NULL, &intr->vmci_handler);
		if (error)
			return (error);
		bus_describe_intr(sc->vmci_dev, intr->vmci_irq,
		    intr->vmci_handler, "bm");
	}

	return (0);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_interrupt --
 *
 *     Interrupt handler for legacy or MSI interrupt, or for first MSI-X
 *     interrupt (vector VMCI_INTR_DATAGRAM).
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_interrupt(void *arg)
{

	if (vmci_sc->vmci_num_intr == 2)
		taskqueue_enqueue(taskqueue_swi,
		    &vmci_sc->vmci_interrupt_dq_task);
	else {
		unsigned int icr;

		icr = inl(vmci_sc->vmci_ioaddr + VMCI_ICR_ADDR);
		if (icr == 0 || icr == 0xffffffff)
			return;
		if (icr & VMCI_ICR_DATAGRAM) {
			taskqueue_enqueue(taskqueue_swi,
			    &vmci_sc->vmci_interrupt_dq_task);
			icr &= ~VMCI_ICR_DATAGRAM;
		}
		if (icr & VMCI_ICR_NOTIFICATION) {
			taskqueue_enqueue(taskqueue_swi,
			    &vmci_sc->vmci_interrupt_bm_task);
			icr &= ~VMCI_ICR_NOTIFICATION;
		}
		if (icr != 0)
			VMCI_LOG_INFO(LGPFX"Ignoring unknown interrupt "
			    "cause");
	}
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_interrupt_bm --
 *
 *     Interrupt handler for MSI-X interrupt vector VMCI_INTR_NOTIFICATION,
 *     which is for the notification bitmap. Will only get called if we are
 *     using MSI-X with exclusive vectors.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_interrupt_bm(void *arg)
{

	ASSERT(vmci_sc->vmci_num_intr == 2);
	taskqueue_enqueue(taskqueue_swi, &vmci_sc->vmci_interrupt_bm_task);
}

/*
 *------------------------------------------------------------------------------
 *
 * dispatch_datagrams --
 *
 *     Reads and dispatches incoming datagrams.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     Reads data from the device.
 *
 *------------------------------------------------------------------------------
 */

static void
dispatch_datagrams(void *context, int data)
{

	if (data_buffer == NULL)
		VMCI_LOG_INFO(LGPFX"dispatch_datagrams(): no buffer "
		    "present");

	vmci_read_datagrams_from_port((vmci_io_handle) 0,
	    vmci_sc->vmci_ioaddr + VMCI_DATA_IN_ADDR,
	    data_buffer, data_buffer_size);
}

/*
 *------------------------------------------------------------------------------
 *
 * process_bitmap --
 *
 *     Scans the notification bitmap for raised flags, clears them and handles
 *     the notifications.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
process_bitmap(void *context, int data)
{

	if (vmci_sc->vmci_notifications_bitmap.dma_vaddr == NULL)
		VMCI_LOG_INFO(LGPFX"process_bitmaps(): no bitmap present");

	vmci_scan_notification_bitmap(
	    vmci_sc->vmci_notifications_bitmap.dma_vaddr);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_dismantle_interrupts --
 *
 *     Releases resources, detaches the interrupt handler and drains the task
 *     queue.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     No more interrupts.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_dismantle_interrupts(struct vmci_softc *sc)
{
	struct vmci_interrupt *intr;
	int i;

	for (i = 0; i < sc->vmci_num_intr; i++) {
		intr = &sc->vmci_intrs[i];
		if (intr->vmci_handler != NULL) {
			bus_teardown_intr(sc->vmci_dev, intr->vmci_irq,
			    intr->vmci_handler);
			intr->vmci_handler = NULL;
		}
		if (intr->vmci_irq != NULL) {
			bus_release_resource(sc->vmci_dev, SYS_RES_IRQ,
			    intr->vmci_rid, intr->vmci_irq);
			intr->vmci_irq = NULL;
			intr->vmci_rid = -1;
		}
	}

	if ((sc->vmci_intr_type != VMCI_INTR_TYPE_INTX) &&
	    (sc->vmci_num_intr))
		pci_release_msi(sc->vmci_dev);

	taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_dq_task);
	taskqueue_drain(taskqueue_swi, &sc->vmci_interrupt_bm_task);

	if (data_buffer != NULL)
		free(data_buffer, M_DEVBUF);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_delayed_work_fn_cb --
 *
 *     Callback function that executes the queued up delayed work functions.
 *
 * Results:
 *     None.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

static void
vmci_delayed_work_fn_cb(void *context, int data)
{
	vmci_list(vmci_delayed_work_info) temp_list;

	vmci_list_init(&temp_list);

	/*
	 * Swap vmci_delayed_work_infos list with the empty temp_list while
	 * holding a lock. vmci_delayed_work_infos would then be an empty list
	 * and temp_list would contain the elements from the original
	 * vmci_delayed_work_infos. Finally, iterate through temp_list
	 * executing the delayed callbacks.
	 */

	mtx_lock(&vmci_sc->vmci_delayed_work_lock);
	vmci_list_swap(&temp_list, &vmci_sc->vmci_delayed_work_infos,
	    vmci_delayed_work_info, entry);
	mtx_unlock(&vmci_sc->vmci_delayed_work_lock);

	while (!vmci_list_empty(&temp_list)) {
		struct vmci_delayed_work_info *delayed_work_info =
		    vmci_list_first(&temp_list);

		delayed_work_info->work_fn(delayed_work_info->data);

		vmci_list_remove(delayed_work_info, entry);
		vmci_free_kernel_mem(delayed_work_info,
		    sizeof(*delayed_work_info));
	}
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_schedule_delayed_work_fn --
 *
 *     Schedule the specified callback.
 *
 * Results:
 *     0 if success, error code otherwise.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

int
vmci_schedule_delayed_work_fn(vmci_work_fn *work_fn, void *data)
{
	struct vmci_delayed_work_info *delayed_work_info;

	delayed_work_info = vmci_alloc_kernel_mem(sizeof(*delayed_work_info),
	    VMCI_MEMORY_ATOMIC);

	if (!delayed_work_info)
		return (VMCI_ERROR_NO_MEM);

	delayed_work_info->work_fn = work_fn;
	delayed_work_info->data = data;
	mtx_lock(&vmci_sc->vmci_delayed_work_lock);
	vmci_list_insert(&vmci_sc->vmci_delayed_work_infos,
	    delayed_work_info, entry);
	mtx_unlock(&vmci_sc->vmci_delayed_work_lock);

	taskqueue_enqueue(taskqueue_thread,
	    &vmci_sc->vmci_delayed_work_task);

	return (VMCI_SUCCESS);
}

/*
 *------------------------------------------------------------------------------
 *
 * vmci_send_datagram --
 *
 *     VM to hypervisor call mechanism.
 *
 * Results:
 *     The result of the hypercall.
 *
 * Side effects:
 *     None.
 *
 *------------------------------------------------------------------------------
 */

int
vmci_send_datagram(struct vmci_datagram *dg)
{
	int result;

	if (dg == NULL)
		return (VMCI_ERROR_INVALID_ARGS);

	/*
	 * Need to acquire spinlock on the device because
	 * the datagram data may be spread over multiple pages and the monitor
	 * may interleave device user rpc calls from multiple VCPUs. Acquiring
	 * the spinlock precludes that possibility. Disabling interrupts to
	 * avoid incoming datagrams during a "rep out" and possibly landing up
	 * in this function.
	 */
	mtx_lock_spin(&vmci_sc->vmci_spinlock);

	/*
	 * Send the datagram and retrieve the return value from the result
	 * register.
	 */
	__asm__ __volatile__(
	    "cld\n\t"
	    "rep outsb\n\t"
	    : /* No output. */
	    : "d"(vmci_sc->vmci_ioaddr + VMCI_DATA_OUT_ADDR),
	    "c"(VMCI_DG_SIZE(dg)), "S"(dg)
	    );

	/*
	 * XXX: Should read result high port as well when updating handlers to
	 * return 64bit.
	 */

	result = bus_space_read_4(vmci_sc->vmci_iot0,
	    vmci_sc->vmci_ioh0, VMCI_RESULT_LOW_ADDR);
	mtx_unlock_spin(&vmci_sc->vmci_spinlock);

	return (result);
}