xref: /freebsd/sys/dev/virtio/console/virtio_console.c (revision 018a361f8902d629fca64be372ef9b266fee53ba)

/*-
 * Copyright (c) 2014, Bryan Venteicher <bryanv@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/* Driver for VirtIO console devices. */

#include <sys/param.h>
#include <sys/ctype.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/kdb.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sglist.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/queue.h>

#include <sys/conf.h>
#include <sys/cons.h>
#include <sys/tty.h>

#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>

#include <dev/virtio/virtio.h>
#include <dev/virtio/virtqueue.h>
#include <dev/virtio/console/virtio_console.h>

#include "virtio_if.h"

#define VTCON_MAX_PORTS 32
#define VTCON_TTY_PREFIX "V"
#define VTCON_TTY_ALIAS_PREFIX "vtcon"
#define VTCON_BULK_BUFSZ 128
#define VTCON_CTRL_BUFSZ 128

/*
 * The buffers cannot cross more than one page boundary due to the
 * size of the sglist segment array used.
 */
CTASSERT(VTCON_BULK_BUFSZ <= PAGE_SIZE);
CTASSERT(VTCON_CTRL_BUFSZ <= PAGE_SIZE);

CTASSERT(sizeof(struct virtio_console_config) <= VTCON_CTRL_BUFSZ);

struct vtcon_softc;
struct vtcon_softc_port;

struct vtcon_port {
	struct mtx			 vtcport_mtx;
	struct vtcon_softc		*vtcport_sc;
	struct vtcon_softc_port		*vtcport_scport;
	struct tty			*vtcport_tty;
	struct virtqueue		*vtcport_invq;
	struct virtqueue		*vtcport_outvq;
	int				 vtcport_id;
	int				 vtcport_flags;
#define VTCON_PORT_FLAG_GONE	0x01
#define VTCON_PORT_FLAG_CONSOLE	0x02
#define VTCON_PORT_FLAG_ALIAS	0x04

#if defined(KDB)
	int				 vtcport_alt_break_state;
#endif
};

#define VTCON_PORT_LOCK(_port)		mtx_lock(&(_port)->vtcport_mtx)
#define VTCON_PORT_UNLOCK(_port)	mtx_unlock(&(_port)->vtcport_mtx)

struct vtcon_softc_port {
	struct vtcon_softc	*vcsp_sc;
	struct vtcon_port	*vcsp_port;
	struct virtqueue	*vcsp_invq;
	struct virtqueue	*vcsp_outvq;
};

struct vtcon_softc {
	device_t		 vtcon_dev;
	struct mtx		 vtcon_mtx;
	uint64_t		 vtcon_features;
	uint32_t		 vtcon_max_ports;
	uint32_t		 vtcon_flags;
#define VTCON_FLAG_DETACHED	0x01
#define VTCON_FLAG_SIZE		0x02
#define VTCON_FLAG_MULTIPORT	0x04

	/*
	 * Ports can be added and removed during runtime, but we have
	 * to allocate all the virtqueues during attach. This array is
	 * indexed by the port ID.
	 */
	struct vtcon_softc_port	*vtcon_ports;

	struct task		 vtcon_ctrl_task;
	struct virtqueue	*vtcon_ctrl_rxvq;
	struct virtqueue	*vtcon_ctrl_txvq;
	struct mtx		 vtcon_ctrl_tx_mtx;
};

#define VTCON_LOCK(_sc)			mtx_lock(&(_sc)->vtcon_mtx)
#define VTCON_UNLOCK(_sc)		mtx_unlock(&(_sc)->vtcon_mtx)
#define VTCON_LOCK_ASSERT(_sc)		\
    mtx_assert(&(_sc)->vtcon_mtx, MA_OWNED)
#define VTCON_LOCK_ASSERT_NOTOWNED(_sc)	\
    mtx_assert(&(_sc)->vtcon_mtx, MA_NOTOWNED)

#define VTCON_CTRL_TX_LOCK(_sc)		mtx_lock(&(_sc)->vtcon_ctrl_tx_mtx)
#define VTCON_CTRL_TX_UNLOCK(_sc)	mtx_unlock(&(_sc)->vtcon_ctrl_tx_mtx)

#define VTCON_ASSERT_VALID_PORTID(_sc, _id)			\
    KASSERT((_id) >= 0 && (_id) < (_sc)->vtcon_max_ports,	\
        ("%s: port ID %d out of range", __func__, _id))

#define VTCON_FEATURES  VIRTIO_CONSOLE_F_MULTIPORT

static struct virtio_feature_desc vtcon_feature_desc[] = {
	{ VIRTIO_CONSOLE_F_SIZE,	"ConsoleSize"	},
	{ VIRTIO_CONSOLE_F_MULTIPORT,	"MultiplePorts"	},
	{ VIRTIO_CONSOLE_F_EMERG_WRITE,	"EmergencyWrite" },
	{ 0, NULL }
};

static int	 vtcon_modevent(module_t, int, void *);
static void	 vtcon_drain_all(void);

static int	 vtcon_probe(device_t);
static int	 vtcon_attach(device_t);
static int	 vtcon_detach(device_t);
static int	 vtcon_config_change(device_t);

static int	 vtcon_setup_features(struct vtcon_softc *);
static int	 vtcon_negotiate_features(struct vtcon_softc *);
static int	 vtcon_alloc_scports(struct vtcon_softc *);
static int	 vtcon_alloc_virtqueues(struct vtcon_softc *);
static void	 vtcon_read_config(struct vtcon_softc *,
		     struct virtio_console_config *);

static void	 vtcon_determine_max_ports(struct vtcon_softc *,
		     struct virtio_console_config *);
static void	 vtcon_destroy_ports(struct vtcon_softc *);
static void	 vtcon_stop(struct vtcon_softc *);

static int	 vtcon_ctrl_event_enqueue(struct vtcon_softc *,
		     struct virtio_console_control *);
static int	 vtcon_ctrl_event_create(struct vtcon_softc *);
static void	 vtcon_ctrl_event_requeue(struct vtcon_softc *,
		     struct virtio_console_control *);
static int	 vtcon_ctrl_event_populate(struct vtcon_softc *);
static void	 vtcon_ctrl_event_drain(struct vtcon_softc *);
static int	 vtcon_ctrl_init(struct vtcon_softc *);
static void	 vtcon_ctrl_deinit(struct vtcon_softc *);
static void	 vtcon_ctrl_port_add_event(struct vtcon_softc *, int);
static void	 vtcon_ctrl_port_remove_event(struct vtcon_softc *, int);
static void	 vtcon_ctrl_port_console_event(struct vtcon_softc *, int);
static void	 vtcon_ctrl_port_open_event(struct vtcon_softc *, int);
static void	 vtcon_ctrl_port_name_event(struct vtcon_softc *, int,
		     const char *, size_t);
static void	 vtcon_ctrl_process_event(struct vtcon_softc *,
		     struct virtio_console_control *, void *, size_t);
static void	 vtcon_ctrl_task_cb(void *, int);
static void	 vtcon_ctrl_event_intr(void *);
static void	 vtcon_ctrl_poll(struct vtcon_softc *,
		     struct virtio_console_control *control);
static void	 vtcon_ctrl_send_control(struct vtcon_softc *, uint32_t,
		     uint16_t, uint16_t);

static int	 vtcon_port_enqueue_buf(struct vtcon_port *, void *, size_t);
static int	 vtcon_port_create_buf(struct vtcon_port *);
static void	 vtcon_port_requeue_buf(struct vtcon_port *, void *);
static int	 vtcon_port_populate(struct vtcon_port *);
static void	 vtcon_port_destroy(struct vtcon_port *);
static int	 vtcon_port_create(struct vtcon_softc *, int);
static void	 vtcon_port_dev_alias(struct vtcon_port *, const char *,
		     size_t);
static void	 vtcon_port_drain_bufs(struct virtqueue *);
static void	 vtcon_port_drain(struct vtcon_port *);
static void	 vtcon_port_teardown(struct vtcon_port *);
static void	 vtcon_port_change_size(struct vtcon_port *, uint16_t,
		     uint16_t);
static void	 vtcon_port_update_console_size(struct vtcon_softc *);
static void	 vtcon_port_enable_intr(struct vtcon_port *);
static void	 vtcon_port_disable_intr(struct vtcon_port *);
static void	 vtcon_port_in(struct vtcon_port *);
static void	 vtcon_port_intr(void *);
static void	 vtcon_port_out(struct vtcon_port *, void *, int);
static void	 vtcon_port_submit_event(struct vtcon_port *, uint16_t,
		     uint16_t);

static int	 vtcon_tty_open(struct tty *);
static void	 vtcon_tty_close(struct tty *);
static void	 vtcon_tty_outwakeup(struct tty *);
static void	 vtcon_tty_free(void *);

static void	 vtcon_get_console_size(struct vtcon_softc *, uint16_t *,
		     uint16_t *);

static void	 vtcon_enable_interrupts(struct vtcon_softc *);
static void	 vtcon_disable_interrupts(struct vtcon_softc *);

#define vtcon_modern(_sc) (((_sc)->vtcon_features & VIRTIO_F_VERSION_1) != 0)
#define vtcon_htog16(_sc, _val)	virtio_htog16(vtcon_modern(_sc), _val)
#define vtcon_htog32(_sc, _val)	virtio_htog32(vtcon_modern(_sc), _val)
#define vtcon_htog64(_sc, _val)	virtio_htog64(vtcon_modern(_sc), _val)
#define vtcon_gtoh16(_sc, _val)	virtio_gtoh16(vtcon_modern(_sc), _val)
#define vtcon_gtoh32(_sc, _val)	virtio_gtoh32(vtcon_modern(_sc), _val)
#define vtcon_gtoh64(_sc, _val)	virtio_gtoh64(vtcon_modern(_sc), _val)

static int	 vtcon_pending_free;

static struct ttydevsw vtcon_tty_class = {
	.tsw_flags	= 0,
	.tsw_open	= vtcon_tty_open,
	.tsw_close	= vtcon_tty_close,
	.tsw_outwakeup	= vtcon_tty_outwakeup,
	.tsw_free	= vtcon_tty_free,
};

static device_method_t vtcon_methods[] = {
	/* Device methods. */
	DEVMETHOD(device_probe,		vtcon_probe),
	DEVMETHOD(device_attach,	vtcon_attach),
	DEVMETHOD(device_detach,	vtcon_detach),

	/* VirtIO methods. */
	DEVMETHOD(virtio_config_change,	vtcon_config_change),

	DEVMETHOD_END
};

static driver_t vtcon_driver = {
	"vtcon",
	vtcon_methods,
	sizeof(struct vtcon_softc)
};

VIRTIO_DRIVER_MODULE(virtio_console, vtcon_driver, vtcon_modevent, NULL);
MODULE_VERSION(virtio_console, 1);
MODULE_DEPEND(virtio_console, virtio, 1, 1, 1);

VIRTIO_SIMPLE_PNPINFO(virtio_console, VIRTIO_ID_CONSOLE,
    "VirtIO Console Adapter");

static int
vtcon_modevent(module_t mod, int type, void *unused)
{
	int error;

	switch (type) {
	case MOD_LOAD:
		error = 0;
		break;
	case MOD_QUIESCE:
		error = 0;
		break;
	case MOD_UNLOAD:
		vtcon_drain_all();
		error = 0;
		break;
	case MOD_SHUTDOWN:
		error = 0;
		break;
	default:
		error = EOPNOTSUPP;
		break;
	}

	return (error);
}

static void
vtcon_drain_all(void)
{
	int first;

	for (first = 1; vtcon_pending_free != 0; first = 0) {
		if (first != 0) {
			printf("virtio_console: Waiting for all detached TTY "
			    "devices to have open fds closed.\n");
		}
		pause("vtcondra", hz);
	}
}

static int
vtcon_probe(device_t dev)
{
	return (VIRTIO_SIMPLE_PROBE(dev, virtio_console));
}

static int
vtcon_attach(device_t dev)
{
	struct vtcon_softc *sc;
	struct virtio_console_config concfg;
	int error;

	sc = device_get_softc(dev);
	sc->vtcon_dev = dev;
	virtio_set_feature_desc(dev, vtcon_feature_desc);

	mtx_init(&sc->vtcon_mtx, "vtconmtx", NULL, MTX_DEF);
	mtx_init(&sc->vtcon_ctrl_tx_mtx, "vtconctrlmtx", NULL, MTX_DEF);

	error = vtcon_setup_features(sc);
	if (error) {
		device_printf(dev, "cannot setup features\n");
		goto fail;
	}

	vtcon_read_config(sc, &concfg);
	vtcon_determine_max_ports(sc, &concfg);

	error = vtcon_alloc_scports(sc);
	if (error) {
		device_printf(dev, "cannot allocate softc port structures\n");
		goto fail;
	}

	error = vtcon_alloc_virtqueues(sc);
	if (error) {
		device_printf(dev, "cannot allocate virtqueues\n");
		goto fail;
	}

	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
		TASK_INIT(&sc->vtcon_ctrl_task, 0, vtcon_ctrl_task_cb, sc);
		error = vtcon_ctrl_init(sc);
		if (error)
			goto fail;
	} else {
		error = vtcon_port_create(sc, 0);
		if (error)
			goto fail;
		if (sc->vtcon_flags & VTCON_FLAG_SIZE)
			vtcon_port_update_console_size(sc);
	}

	error = virtio_setup_intr(dev, INTR_TYPE_TTY);
	if (error) {
		device_printf(dev, "cannot setup virtqueue interrupts\n");
		goto fail;
	}

	vtcon_enable_interrupts(sc);

	vtcon_ctrl_send_control(sc, VIRTIO_CONSOLE_BAD_ID,
	    VIRTIO_CONSOLE_DEVICE_READY, 1);

fail:
	if (error)
		vtcon_detach(dev);

	return (error);
}

static int
vtcon_detach(device_t dev)
{
	struct vtcon_softc *sc;

	sc = device_get_softc(dev);

	VTCON_LOCK(sc);
	sc->vtcon_flags |= VTCON_FLAG_DETACHED;
	if (device_is_attached(dev))
		vtcon_stop(sc);
	VTCON_UNLOCK(sc);

	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
		taskqueue_drain(taskqueue_thread, &sc->vtcon_ctrl_task);
		vtcon_ctrl_deinit(sc);
	}

	vtcon_destroy_ports(sc);
	mtx_destroy(&sc->vtcon_mtx);
	mtx_destroy(&sc->vtcon_ctrl_tx_mtx);

	return (0);
}

static int
vtcon_config_change(device_t dev)
{
	struct vtcon_softc *sc;

	sc = device_get_softc(dev);

	/*
	 * When the multiport feature is negotiated, all configuration
	 * changes are done through control virtqueue events.
	 */
	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0) {
		if (sc->vtcon_flags & VTCON_FLAG_SIZE)
			vtcon_port_update_console_size(sc);
	}

	return (0);
}

static int
vtcon_negotiate_features(struct vtcon_softc *sc)
{
	device_t dev;
	uint64_t features;

	dev = sc->vtcon_dev;
	features = VTCON_FEATURES;

	sc->vtcon_features = virtio_negotiate_features(dev, features);
	return (virtio_finalize_features(dev));
}

static int
vtcon_setup_features(struct vtcon_softc *sc)
{
	device_t dev;
	int error;

	dev = sc->vtcon_dev;

	error = vtcon_negotiate_features(sc);
	if (error)
		return (error);

	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_SIZE))
		sc->vtcon_flags |= VTCON_FLAG_SIZE;
	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_MULTIPORT))
		sc->vtcon_flags |= VTCON_FLAG_MULTIPORT;

	return (0);
}

#define VTCON_GET_CONFIG(_dev, _feature, _field, _cfg)			\
	if (virtio_with_feature(_dev, _feature)) {			\
		virtio_read_device_config(_dev,				\
		    offsetof(struct virtio_console_config, _field),	\
		    &(_cfg)->_field, sizeof((_cfg)->_field));		\
	}

static void
vtcon_read_config(struct vtcon_softc *sc, struct virtio_console_config *concfg)
{
	device_t dev;

	dev = sc->vtcon_dev;

	bzero(concfg, sizeof(struct virtio_console_config));

	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, cols, concfg);
	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, rows, concfg);
	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_MULTIPORT, max_nr_ports, concfg);
}

#undef VTCON_GET_CONFIG

static int
vtcon_alloc_scports(struct vtcon_softc *sc)
{
	struct vtcon_softc_port *scport;
	int max, i;

	max = sc->vtcon_max_ports;

	sc->vtcon_ports = malloc(sizeof(struct vtcon_softc_port) * max,
	    M_DEVBUF, M_NOWAIT | M_ZERO);
	if (sc->vtcon_ports == NULL)
		return (ENOMEM);

	for (i = 0; i < max; i++) {
		scport = &sc->vtcon_ports[i];
		scport->vcsp_sc = sc;
	}

	return (0);
}

static int
vtcon_alloc_virtqueues(struct vtcon_softc *sc)
{
	device_t dev;
	struct vq_alloc_info *info;
	struct vtcon_softc_port *scport;
	int i, idx, portidx, nvqs, error;

	dev = sc->vtcon_dev;

	nvqs = sc->vtcon_max_ports * 2;
	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
		nvqs += 2;

	info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT);
	if (info == NULL)
		return (ENOMEM);

	for (i = 0, idx = 0, portidx = 0; i < nvqs / 2; i++, idx += 2) {
		if (i == 1) {
			/* The control virtqueues are after the first port. */
			VQ_ALLOC_INFO_INIT(&info[idx], 0,
			    vtcon_ctrl_event_intr, sc, &sc->vtcon_ctrl_rxvq,
			    "%s-control rx", device_get_nameunit(dev));
			VQ_ALLOC_INFO_INIT(&info[idx+1], 0,
			    NULL, sc, &sc->vtcon_ctrl_txvq,
			    "%s-control tx", device_get_nameunit(dev));
			continue;
		}

		scport = &sc->vtcon_ports[portidx];

		VQ_ALLOC_INFO_INIT(&info[idx], 0, vtcon_port_intr,
		    scport, &scport->vcsp_invq, "%s-port%d in",
		    device_get_nameunit(dev), i);
		VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL,
		    NULL, &scport->vcsp_outvq, "%s-port%d out",
		    device_get_nameunit(dev), i);

		portidx++;
	}

	error = virtio_alloc_virtqueues(dev, nvqs, info);
	free(info, M_TEMP);

	return (error);
}

static void
vtcon_determine_max_ports(struct vtcon_softc *sc,
    struct virtio_console_config *concfg)
{

	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
		sc->vtcon_max_ports =
		    min(concfg->max_nr_ports, VTCON_MAX_PORTS);
		if (sc->vtcon_max_ports == 0)
			sc->vtcon_max_ports = 1;
	} else
		sc->vtcon_max_ports = 1;
}

static void
vtcon_destroy_ports(struct vtcon_softc *sc)
{
	struct vtcon_softc_port *scport;
	struct vtcon_port *port;
	struct virtqueue *vq;
	int i;

	if (sc->vtcon_ports == NULL)
		return;

	VTCON_LOCK(sc);
	for (i = 0; i < sc->vtcon_max_ports; i++) {
		scport = &sc->vtcon_ports[i];

		port = scport->vcsp_port;
		if (port != NULL) {
			scport->vcsp_port = NULL;
			VTCON_PORT_LOCK(port);
			VTCON_UNLOCK(sc);
			vtcon_port_teardown(port);
			VTCON_LOCK(sc);
		}

		vq = scport->vcsp_invq;
		if (vq != NULL)
			vtcon_port_drain_bufs(vq);
	}
	VTCON_UNLOCK(sc);

	free(sc->vtcon_ports, M_DEVBUF);
	sc->vtcon_ports = NULL;
}

static void
vtcon_stop(struct vtcon_softc *sc)
{

	vtcon_disable_interrupts(sc);
	virtio_stop(sc->vtcon_dev);
}

static int
vtcon_ctrl_event_enqueue(struct vtcon_softc *sc,
    struct virtio_console_control *control)
{
	struct sglist_seg segs[2];
	struct sglist sg;
	struct virtqueue *vq;
	int error __diagused;

	vq = sc->vtcon_ctrl_rxvq;

	sglist_init(&sg, 2, segs);
	error = sglist_append(&sg, control, VTCON_CTRL_BUFSZ);
	KASSERT(error == 0, ("%s: error %d adding control to sglist",
	    __func__, error));

	return (virtqueue_enqueue(vq, control, &sg, 0, sg.sg_nseg));
}

static int
vtcon_ctrl_event_create(struct vtcon_softc *sc)
{
	struct virtio_console_control *control;
	int error;

	control = malloc(VTCON_CTRL_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT);
	if (control == NULL)
		return (ENOMEM);

	error = vtcon_ctrl_event_enqueue(sc, control);
	if (error)
		free(control, M_DEVBUF);

	return (error);
}

static void
vtcon_ctrl_event_requeue(struct vtcon_softc *sc,
    struct virtio_console_control *control)
{
	int error __diagused;

	bzero(control, VTCON_CTRL_BUFSZ);

	error = vtcon_ctrl_event_enqueue(sc, control);
	KASSERT(error == 0,
	    ("%s: cannot requeue control buffer %d", __func__, error));
}

static int
vtcon_ctrl_event_populate(struct vtcon_softc *sc)
{
	struct virtqueue *vq;
	int nbufs, error;

	vq = sc->vtcon_ctrl_rxvq;
	error = ENOSPC;

	for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
		error = vtcon_ctrl_event_create(sc);
		if (error)
			break;
	}

	if (nbufs > 0) {
		virtqueue_notify(vq);
		error = 0;
	}

	return (error);
}

static void
vtcon_ctrl_event_drain(struct vtcon_softc *sc)
{
	struct virtio_console_control *control;
	struct virtqueue *vq;
	int last;

	vq = sc->vtcon_ctrl_rxvq;
	last = 0;

	if (vq == NULL)
		return;

	VTCON_LOCK(sc);
	while ((control = virtqueue_drain(vq, &last)) != NULL)
		free(control, M_DEVBUF);
	VTCON_UNLOCK(sc);
}

static int
vtcon_ctrl_init(struct vtcon_softc *sc)
{
	int error;

	error = vtcon_ctrl_event_populate(sc);

	return (error);
}

static void
vtcon_ctrl_deinit(struct vtcon_softc *sc)
{

	vtcon_ctrl_event_drain(sc);
}

static void
vtcon_ctrl_port_add_event(struct vtcon_softc *sc, int id)
{
	device_t dev;
	int error;

	dev = sc->vtcon_dev;

	/* This single thread only way for ports to be created. */
	if (sc->vtcon_ports[id].vcsp_port != NULL) {
		device_printf(dev, "%s: adding port %d, but already exists\n",
		    __func__, id);
		return;
	}

	error = vtcon_port_create(sc, id);
	if (error) {
		device_printf(dev, "%s: cannot create port %d: %d\n",
		    __func__, id, error);
		vtcon_ctrl_send_control(sc, id, VIRTIO_CONSOLE_PORT_READY, 0);
		return;
	}
}

static void
vtcon_ctrl_port_remove_event(struct vtcon_softc *sc, int id)
{
	device_t dev;
	struct vtcon_softc_port *scport;
	struct vtcon_port *port;

	dev = sc->vtcon_dev;
	scport = &sc->vtcon_ports[id];

	VTCON_LOCK(sc);
	port = scport->vcsp_port;
	if (port == NULL) {
		VTCON_UNLOCK(sc);
		device_printf(dev, "%s: remove port %d, but does not exist\n",
		    __func__, id);
		return;
	}

	scport->vcsp_port = NULL;
	VTCON_PORT_LOCK(port);
	VTCON_UNLOCK(sc);
	vtcon_port_teardown(port);
}

static void
vtcon_ctrl_port_console_event(struct vtcon_softc *sc, int id)
{
	device_t dev;
	struct vtcon_softc_port *scport;
	struct vtcon_port *port;

	dev = sc->vtcon_dev;
	scport = &sc->vtcon_ports[id];

	VTCON_LOCK(sc);
	port = scport->vcsp_port;
	if (port == NULL) {
		VTCON_UNLOCK(sc);
		device_printf(dev, "%s: console port %d, but does not exist\n",
		    __func__, id);
		return;
	}

	VTCON_PORT_LOCK(port);
	VTCON_UNLOCK(sc);
	port->vtcport_flags |= VTCON_PORT_FLAG_CONSOLE;
	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
	VTCON_PORT_UNLOCK(port);
}

static void
vtcon_ctrl_port_open_event(struct vtcon_softc *sc, int id)
{
	device_t dev;
	struct vtcon_softc_port *scport;
	struct vtcon_port *port;

	dev = sc->vtcon_dev;
	scport = &sc->vtcon_ports[id];

	VTCON_LOCK(sc);
	port = scport->vcsp_port;
	if (port == NULL) {
		VTCON_UNLOCK(sc);
		device_printf(dev, "%s: open port %d, but does not exist\n",
		    __func__, id);
		return;
	}

	VTCON_PORT_LOCK(port);
	VTCON_UNLOCK(sc);
	vtcon_port_enable_intr(port);
	VTCON_PORT_UNLOCK(port);
}

static void
vtcon_ctrl_port_name_event(struct vtcon_softc *sc, int id, const char *name,
    size_t len)
{
	device_t dev;
	struct vtcon_softc_port *scport;
	struct vtcon_port *port;

	dev = sc->vtcon_dev;
	scport = &sc->vtcon_ports[id];

	/*
	 * The VirtIO specification says the NUL terminator is not included in
	 * the length, but QEMU includes it. Adjust the length if needed.
	 */
	if (name == NULL || len == 0)
		return;
	if (name[len - 1] == '\0') {
		len--;
		if (len == 0)
			return;
	}

	VTCON_LOCK(sc);
	port = scport->vcsp_port;
	if (port == NULL) {
		VTCON_UNLOCK(sc);
		device_printf(dev, "%s: name port %d, but does not exist\n",
		    __func__, id);
		return;
	}

	VTCON_PORT_LOCK(port);
	VTCON_UNLOCK(sc);
	vtcon_port_dev_alias(port, name, len);
	VTCON_PORT_UNLOCK(port);
}

static void
vtcon_ctrl_process_event(struct vtcon_softc *sc,
    struct virtio_console_control *control, void *data, size_t data_len)
{
	device_t dev;
	uint32_t id;
	uint16_t event;

	dev = sc->vtcon_dev;
	id = vtcon_htog32(sc, control->id);
	event = vtcon_htog16(sc, control->event);

	if (id >= sc->vtcon_max_ports) {
		device_printf(dev, "%s: event %d invalid port ID %d\n",
		    __func__, event, id);
		return;
	}

	switch (event) {
	case VIRTIO_CONSOLE_PORT_ADD:
		vtcon_ctrl_port_add_event(sc, id);
		break;

	case VIRTIO_CONSOLE_PORT_REMOVE:
		vtcon_ctrl_port_remove_event(sc, id);
		break;

	case VIRTIO_CONSOLE_CONSOLE_PORT:
		vtcon_ctrl_port_console_event(sc, id);
		break;

	case VIRTIO_CONSOLE_RESIZE:
		break;

	case VIRTIO_CONSOLE_PORT_OPEN:
		vtcon_ctrl_port_open_event(sc, id);
		break;

	case VIRTIO_CONSOLE_PORT_NAME:
		vtcon_ctrl_port_name_event(sc, id, (const char *)data, data_len);
		break;
	}
}

static void
vtcon_ctrl_task_cb(void *xsc, int pending)
{
	struct vtcon_softc *sc;
	struct virtqueue *vq;
	struct virtio_console_control *control;
	void *data;
	size_t data_len;
	int detached;
	uint32_t len;

	sc = xsc;
	vq = sc->vtcon_ctrl_rxvq;

	VTCON_LOCK(sc);

	while ((detached = (sc->vtcon_flags & VTCON_FLAG_DETACHED)) == 0) {
		control = virtqueue_dequeue(vq, &len);
		if (control == NULL)
			break;

		if (len > sizeof(struct virtio_console_control)) {
			data = (void *) &control[1];
			data_len = len - sizeof(struct virtio_console_control);
		} else {
			data = NULL;
			data_len = 0;
		}

		VTCON_UNLOCK(sc);
		vtcon_ctrl_process_event(sc, control, data, data_len);
		VTCON_LOCK(sc);
		vtcon_ctrl_event_requeue(sc, control);
	}

	if (!detached) {
		virtqueue_notify(vq);
		if (virtqueue_enable_intr(vq) != 0)
			taskqueue_enqueue(taskqueue_thread,
			    &sc->vtcon_ctrl_task);
	}

	VTCON_UNLOCK(sc);
}

static void
vtcon_ctrl_event_intr(void *xsc)
{
	struct vtcon_softc *sc;

	sc = xsc;

	/*
	 * Only some events require us to potentially block, but it
	 * easier to just defer all event handling to the taskqueue.
	 */
	taskqueue_enqueue(taskqueue_thread, &sc->vtcon_ctrl_task);
}

static void
vtcon_ctrl_poll(struct vtcon_softc *sc,
    struct virtio_console_control *control)
{
	struct sglist_seg segs[2];
	struct sglist sg;
	struct virtqueue *vq;
	int error;

	vq = sc->vtcon_ctrl_txvq;

	sglist_init(&sg, 2, segs);
	error = sglist_append(&sg, control,
	    sizeof(struct virtio_console_control));
	KASSERT(error == 0, ("%s: error %d adding control to sglist",
	    __func__, error));

	/*
	 * We cannot use the softc lock to serialize access to this
	 * virtqueue since this is called from the tty layer with the
	 * port lock held. Acquiring the softc would violate our lock
	 * ordering.
	 */
	VTCON_CTRL_TX_LOCK(sc);
	KASSERT(virtqueue_empty(vq),
	    ("%s: virtqueue is not empty", __func__));
	error = virtqueue_enqueue(vq, control, &sg, sg.sg_nseg, 0);
	if (error == 0) {
		virtqueue_notify(vq);
		virtqueue_poll(vq, NULL);
	}
	VTCON_CTRL_TX_UNLOCK(sc);
}

static void
vtcon_ctrl_send_control(struct vtcon_softc *sc, uint32_t portid,
    uint16_t event, uint16_t value)
{
	struct virtio_console_control control;

	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0)
		return;

	control.id = vtcon_gtoh32(sc, portid);
	control.event = vtcon_gtoh16(sc, event);
	control.value = vtcon_gtoh16(sc, value);

	vtcon_ctrl_poll(sc, &control);
}

static int
vtcon_port_enqueue_buf(struct vtcon_port *port, void *buf, size_t len)
{
	struct sglist_seg segs[2];
	struct sglist sg;
	struct virtqueue *vq;
	int error;

	vq = port->vtcport_invq;

	sglist_init(&sg, 2, segs);
	error = sglist_append(&sg, buf, len);
	KASSERT(error == 0,
	    ("%s: error %d adding buffer to sglist", __func__, error));

	error = virtqueue_enqueue(vq, buf, &sg, 0, sg.sg_nseg);

	return (error);
}

static int
vtcon_port_create_buf(struct vtcon_port *port)
{
	void *buf;
	int error;

	buf = malloc(VTCON_BULK_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT);
	if (buf == NULL)
		return (ENOMEM);

	error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ);
	if (error)
		free(buf, M_DEVBUF);

	return (error);
}

static void
vtcon_port_requeue_buf(struct vtcon_port *port, void *buf)
{
	int error __diagused;

	error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ);
	KASSERT(error == 0,
	    ("%s: cannot requeue input buffer %d", __func__, error));
}

static int
vtcon_port_populate(struct vtcon_port *port)
{
	struct virtqueue *vq;
	int nbufs, error;

	vq = port->vtcport_invq;
	error = ENOSPC;

	for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
		error = vtcon_port_create_buf(port);
		if (error)
			break;
	}

	if (nbufs > 0) {
		virtqueue_notify(vq);
		error = 0;
	}

	return (error);
}

static void
vtcon_port_destroy(struct vtcon_port *port)
{

	port->vtcport_sc = NULL;
	port->vtcport_scport = NULL;
	port->vtcport_invq = NULL;
	port->vtcport_outvq = NULL;
	port->vtcport_id = -1;
	mtx_destroy(&port->vtcport_mtx);
	free(port, M_DEVBUF);
}

static int
vtcon_port_init_vqs(struct vtcon_port *port)
{
	struct vtcon_softc_port *scport;
	int error;

	scport = port->vtcport_scport;

	port->vtcport_invq = scport->vcsp_invq;
	port->vtcport_outvq = scport->vcsp_outvq;

	/*
	 * Free any data left over from when this virtqueue was in use by a
	 * prior port. We have not yet notified the host that the port is
	 * ready, so assume nothing in the virtqueue can be for us.
	 */
	vtcon_port_drain(port);

	KASSERT(virtqueue_empty(port->vtcport_invq),
	    ("%s: in virtqueue is not empty", __func__));
	KASSERT(virtqueue_empty(port->vtcport_outvq),
	    ("%s: out virtqueue is not empty", __func__));

	error = vtcon_port_populate(port);
	if (error)
		return (error);

	return (0);
}

static int
vtcon_port_create(struct vtcon_softc *sc, int id)
{
	device_t dev;
	struct vtcon_softc_port *scport;
	struct vtcon_port *port;
	int error;

	dev = sc->vtcon_dev;
	scport = &sc->vtcon_ports[id];

	VTCON_ASSERT_VALID_PORTID(sc, id);
	MPASS(scport->vcsp_port == NULL);

	port = malloc(sizeof(struct vtcon_port), M_DEVBUF, M_NOWAIT | M_ZERO);
	if (port == NULL)
		return (ENOMEM);

	port->vtcport_sc = sc;
	port->vtcport_scport = scport;
	port->vtcport_id = id;
	mtx_init(&port->vtcport_mtx, "vtcpmtx", NULL, MTX_DEF);
	port->vtcport_tty = tty_alloc_mutex(&vtcon_tty_class, port,
	    &port->vtcport_mtx);

	error = vtcon_port_init_vqs(port);
	if (error) {
		VTCON_PORT_LOCK(port);
		vtcon_port_teardown(port);
		return (error);
	}

	VTCON_LOCK(sc);
	VTCON_PORT_LOCK(port);
	scport->vcsp_port = port;
	vtcon_port_enable_intr(port);
	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_READY, 1);
	VTCON_PORT_UNLOCK(port);
	VTCON_UNLOCK(sc);

	tty_makedev(port->vtcport_tty, NULL, "%s%r.%r", VTCON_TTY_PREFIX,
	    device_get_unit(dev), id);

	return (0);
}

static void
vtcon_port_dev_alias(struct vtcon_port *port, const char *name, size_t len)
{
	struct vtcon_softc *sc;
	struct cdev *pdev;
	struct tty *tp;
	int i, error;

	sc = port->vtcport_sc;
	tp = port->vtcport_tty;

	if (port->vtcport_flags & VTCON_PORT_FLAG_ALIAS)
		return;

	/* Port name is UTF-8, but we can only handle ASCII. */
	for (i = 0; i < len; i++) {
		if (!isascii(name[i]))
			return;
	}

	/*
	 * Port name may not conform to the devfs requirements so we cannot use
	 * tty_makealias() because the MAKEDEV_CHECKNAME flag must be specified.
	 */
	error = make_dev_alias_p(MAKEDEV_NOWAIT | MAKEDEV_CHECKNAME, &pdev,
	    tp->t_dev, "%s/%*s", VTCON_TTY_ALIAS_PREFIX, (int)len, name);
	if (error) {
		device_printf(sc->vtcon_dev,
		    "%s: cannot make dev alias (%s/%*s) error %d\n", __func__,
		    VTCON_TTY_ALIAS_PREFIX, (int)len, name, error);
	} else
		port->vtcport_flags |= VTCON_PORT_FLAG_ALIAS;
}

static void
vtcon_port_drain_bufs(struct virtqueue *vq)
{
	void *buf;
	int last;

	last = 0;

	while ((buf = virtqueue_drain(vq, &last)) != NULL)
		free(buf, M_DEVBUF);
}

static void
vtcon_port_drain(struct vtcon_port *port)
{

	vtcon_port_drain_bufs(port->vtcport_invq);
}

static void
vtcon_port_teardown(struct vtcon_port *port)
{
	struct tty *tp;

	tp = port->vtcport_tty;

	port->vtcport_flags |= VTCON_PORT_FLAG_GONE;

	if (tp != NULL) {
		atomic_add_int(&vtcon_pending_free, 1);
		tty_rel_gone(tp);
	} else
		vtcon_port_destroy(port);
}

static void
vtcon_port_change_size(struct vtcon_port *port, uint16_t cols, uint16_t rows)
{
	struct tty *tp;
	struct winsize sz;

	tp = port->vtcport_tty;

	if (tp == NULL)
		return;

	bzero(&sz, sizeof(struct winsize));
	sz.ws_col = cols;
	sz.ws_row = rows;

	tty_set_winsize(tp, &sz);
}

static void
vtcon_port_update_console_size(struct vtcon_softc *sc)
{
	struct vtcon_port *port;
	struct vtcon_softc_port *scport;
	uint16_t cols, rows;

	vtcon_get_console_size(sc, &cols, &rows);

	/*
	 * For now, assume the first (only) port is the console. Note
	 * QEMU does not implement this feature yet.
	 */
	scport = &sc->vtcon_ports[0];

	VTCON_LOCK(sc);
	port = scport->vcsp_port;

	if (port != NULL) {
		VTCON_PORT_LOCK(port);
		VTCON_UNLOCK(sc);
		vtcon_port_change_size(port, cols, rows);
		VTCON_PORT_UNLOCK(port);
	} else
		VTCON_UNLOCK(sc);
}

static void
vtcon_port_enable_intr(struct vtcon_port *port)
{

	/*
	 * NOTE: The out virtqueue is always polled, so its interrupt
	 * kept disabled.
	 */
	virtqueue_enable_intr(port->vtcport_invq);
}

static void
vtcon_port_disable_intr(struct vtcon_port *port)
{

	if (port->vtcport_invq != NULL)
		virtqueue_disable_intr(port->vtcport_invq);
	if (port->vtcport_outvq != NULL)
		virtqueue_disable_intr(port->vtcport_outvq);
}

static void
vtcon_port_in(struct vtcon_port *port)
{
	struct virtqueue *vq;
	struct tty *tp;
	char *buf;
	uint32_t len;
	int i, deq;

	tp = port->vtcport_tty;
	vq = port->vtcport_invq;

again:
	deq = 0;

	while ((buf = virtqueue_dequeue(vq, &len)) != NULL) {
		for (i = 0; i < len; i++) {
#if defined(KDB)
			if (port->vtcport_flags & VTCON_PORT_FLAG_CONSOLE)
				kdb_alt_break(buf[i],
				    &port->vtcport_alt_break_state);
#endif
			ttydisc_rint(tp, buf[i], 0);
		}
		vtcon_port_requeue_buf(port, buf);
		deq++;
	}
	ttydisc_rint_done(tp);

	if (deq > 0)
		virtqueue_notify(vq);

	if (virtqueue_enable_intr(vq) != 0)
		goto again;
}

static void
vtcon_port_intr(void *scportx)
{
	struct vtcon_softc_port *scport;
	struct vtcon_softc *sc;
	struct vtcon_port *port;

	scport = scportx;
	sc = scport->vcsp_sc;

	VTCON_LOCK(sc);
	port = scport->vcsp_port;
	if (port == NULL) {
		VTCON_UNLOCK(sc);
		return;
	}
	VTCON_PORT_LOCK(port);
	VTCON_UNLOCK(sc);
	if ((port->vtcport_flags & VTCON_PORT_FLAG_GONE) == 0)
		vtcon_port_in(port);
	VTCON_PORT_UNLOCK(port);
}

static void
vtcon_port_out(struct vtcon_port *port, void *buf, int bufsize)
{
	struct sglist_seg segs[2];
	struct sglist sg;
	struct virtqueue *vq;
	int error;

	vq = port->vtcport_outvq;
	KASSERT(virtqueue_empty(vq),
	    ("%s: port %p out virtqueue not empty", __func__, port));

	sglist_init(&sg, 2, segs);
	error = sglist_append(&sg, buf, bufsize);
	KASSERT(error == 0, ("%s: error %d adding buffer to sglist",
	    __func__, error));

	error = virtqueue_enqueue(vq, buf, &sg, sg.sg_nseg, 0);
	if (error == 0) {
		virtqueue_notify(vq);
		virtqueue_poll(vq, NULL);
	}
}

static void
vtcon_port_submit_event(struct vtcon_port *port, uint16_t event,
    uint16_t value)
{
	struct vtcon_softc *sc;

	sc = port->vtcport_sc;

	vtcon_ctrl_send_control(sc, port->vtcport_id, event, value);
}

static int
vtcon_tty_open(struct tty *tp)
{
	struct vtcon_port *port;

	port = tty_softc(tp);

	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
		return (ENXIO);

	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);

	return (0);
}

static void
vtcon_tty_close(struct tty *tp)
{
	struct vtcon_port *port;

	port = tty_softc(tp);

	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
		return;

	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 0);
}

static void
vtcon_tty_outwakeup(struct tty *tp)
{
	struct vtcon_port *port;
	char buf[VTCON_BULK_BUFSZ];
	int len;

	port = tty_softc(tp);

	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
		return;

	while ((len = ttydisc_getc(tp, buf, sizeof(buf))) != 0)
		vtcon_port_out(port, buf, len);
}

static void
vtcon_tty_free(void *xport)
{
	struct vtcon_port *port;

	port = xport;

	vtcon_port_destroy(port);
	atomic_subtract_int(&vtcon_pending_free, 1);
}

static void
vtcon_get_console_size(struct vtcon_softc *sc, uint16_t *cols, uint16_t *rows)
{
	struct virtio_console_config concfg;

	KASSERT(sc->vtcon_flags & VTCON_FLAG_SIZE,
	    ("%s: size feature not negotiated", __func__));

	vtcon_read_config(sc, &concfg);

	*cols = concfg.cols;
	*rows = concfg.rows;
}

static void
vtcon_enable_interrupts(struct vtcon_softc *sc)
{
	struct vtcon_softc_port *scport;
	struct vtcon_port *port;
	int i;

	VTCON_LOCK(sc);

	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
		virtqueue_enable_intr(sc->vtcon_ctrl_rxvq);

	for (i = 0; i < sc->vtcon_max_ports; i++) {
		scport = &sc->vtcon_ports[i];

		port = scport->vcsp_port;
		if (port == NULL)
			continue;

		VTCON_PORT_LOCK(port);
		vtcon_port_enable_intr(port);
		VTCON_PORT_UNLOCK(port);
	}

	VTCON_UNLOCK(sc);
}

static void
vtcon_disable_interrupts(struct vtcon_softc *sc)
{
	struct vtcon_softc_port *scport;
	struct vtcon_port *port;
	int i;

	VTCON_LOCK_ASSERT(sc);

	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
		virtqueue_disable_intr(sc->vtcon_ctrl_rxvq);

	for (i = 0; i < sc->vtcon_max_ports; i++) {
		scport = &sc->vtcon_ports[i];

		port = scport->vcsp_port;
		if (port == NULL)
			continue;

		VTCON_PORT_LOCK(port);
		vtcon_port_disable_intr(port);
		VTCON_PORT_UNLOCK(port);
	}
}