xref: /freebsd/sys/dev/virtio/console/virtio_console.c (revision 6e5dcc6113da649a79e5bc2c3ea9329bcd1d85d5)
1 /*-
2  * Copyright (c) 2014, Bryan Venteicher <bryanv@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice unmodified, this list of conditions, and the following
10  *    disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 /* Driver for VirtIO console devices. */
28 
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #include <sys/ctype.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/module.h>
36 #include <sys/kdb.h>
37 #include <sys/lock.h>
38 #include <sys/mutex.h>
39 #include <sys/sglist.h>
40 #include <sys/sysctl.h>
41 #include <sys/taskqueue.h>
42 #include <sys/queue.h>
43 
44 #include <sys/conf.h>
45 #include <sys/cons.h>
46 #include <sys/tty.h>
47 
48 #include <machine/bus.h>
49 #include <machine/resource.h>
50 #include <sys/bus.h>
51 
52 #include <dev/virtio/virtio.h>
53 #include <dev/virtio/virtqueue.h>
54 #include <dev/virtio/console/virtio_console.h>
55 
56 #include "virtio_if.h"
57 
58 #define VTCON_MAX_PORTS 32
59 #define VTCON_TTY_PREFIX "V"
60 #define VTCON_TTY_ALIAS_PREFIX "vtcon"
61 #define VTCON_BULK_BUFSZ 128
62 #define VTCON_CTRL_BUFSZ 128
63 
64 /*
65  * The buffers cannot cross more than one page boundary due to the
66  * size of the sglist segment array used.
67  */
68 CTASSERT(VTCON_BULK_BUFSZ <= PAGE_SIZE);
69 CTASSERT(VTCON_CTRL_BUFSZ <= PAGE_SIZE);
70 
71 CTASSERT(sizeof(struct virtio_console_config) <= VTCON_CTRL_BUFSZ);
72 
73 struct vtcon_softc;
74 struct vtcon_softc_port;
75 
76 struct vtcon_port {
77 	struct mtx			 vtcport_mtx;
78 	struct vtcon_softc		*vtcport_sc;
79 	struct vtcon_softc_port		*vtcport_scport;
80 	struct tty			*vtcport_tty;
81 	struct virtqueue		*vtcport_invq;
82 	struct virtqueue		*vtcport_outvq;
83 	int				 vtcport_id;
84 	int				 vtcport_flags;
85 #define VTCON_PORT_FLAG_GONE	0x01
86 #define VTCON_PORT_FLAG_CONSOLE	0x02
87 #define VTCON_PORT_FLAG_ALIAS	0x04
88 
89 #if defined(KDB)
90 	int				 vtcport_alt_break_state;
91 #endif
92 };
93 
94 #define VTCON_PORT_LOCK(_port)		mtx_lock(&(_port)->vtcport_mtx)
95 #define VTCON_PORT_UNLOCK(_port)	mtx_unlock(&(_port)->vtcport_mtx)
96 
97 struct vtcon_softc_port {
98 	struct vtcon_softc	*vcsp_sc;
99 	struct vtcon_port	*vcsp_port;
100 	struct virtqueue	*vcsp_invq;
101 	struct virtqueue	*vcsp_outvq;
102 };
103 
104 struct vtcon_softc {
105 	device_t		 vtcon_dev;
106 	struct mtx		 vtcon_mtx;
107 	uint64_t		 vtcon_features;
108 	uint32_t		 vtcon_max_ports;
109 	uint32_t		 vtcon_flags;
110 #define VTCON_FLAG_DETACHED	0x01
111 #define VTCON_FLAG_SIZE		0x02
112 #define VTCON_FLAG_MULTIPORT	0x04
113 
114 	/*
115 	 * Ports can be added and removed during runtime, but we have
116 	 * to allocate all the virtqueues during attach. This array is
117 	 * indexed by the port ID.
118 	 */
119 	struct vtcon_softc_port	*vtcon_ports;
120 
121 	struct task		 vtcon_ctrl_task;
122 	struct virtqueue	*vtcon_ctrl_rxvq;
123 	struct virtqueue	*vtcon_ctrl_txvq;
124 	struct mtx		 vtcon_ctrl_tx_mtx;
125 };
126 
127 #define VTCON_LOCK(_sc)			mtx_lock(&(_sc)->vtcon_mtx)
128 #define VTCON_UNLOCK(_sc)		mtx_unlock(&(_sc)->vtcon_mtx)
129 #define VTCON_LOCK_ASSERT(_sc)		\
130     mtx_assert(&(_sc)->vtcon_mtx, MA_OWNED)
131 #define VTCON_LOCK_ASSERT_NOTOWNED(_sc)	\
132     mtx_assert(&(_sc)->vtcon_mtx, MA_NOTOWNED)
133 
134 #define VTCON_CTRL_TX_LOCK(_sc)		mtx_lock(&(_sc)->vtcon_ctrl_tx_mtx)
135 #define VTCON_CTRL_TX_UNLOCK(_sc)	mtx_unlock(&(_sc)->vtcon_ctrl_tx_mtx)
136 
137 #define VTCON_ASSERT_VALID_PORTID(_sc, _id)			\
138     KASSERT((_id) >= 0 && (_id) < (_sc)->vtcon_max_ports,	\
139         ("%s: port ID %d out of range", __func__, _id))
140 
141 #define VTCON_FEATURES  VIRTIO_CONSOLE_F_MULTIPORT
142 
143 static struct virtio_feature_desc vtcon_feature_desc[] = {
144 	{ VIRTIO_CONSOLE_F_SIZE,	"ConsoleSize"	},
145 	{ VIRTIO_CONSOLE_F_MULTIPORT,	"MultiplePorts"	},
146 	{ VIRTIO_CONSOLE_F_EMERG_WRITE,	"EmergencyWrite" },
147 	{ 0, NULL }
148 };
149 
150 static int	 vtcon_modevent(module_t, int, void *);
151 static void	 vtcon_drain_all(void);
152 
153 static int	 vtcon_probe(device_t);
154 static int	 vtcon_attach(device_t);
155 static int	 vtcon_detach(device_t);
156 static int	 vtcon_config_change(device_t);
157 
158 static int	 vtcon_setup_features(struct vtcon_softc *);
159 static int	 vtcon_negotiate_features(struct vtcon_softc *);
160 static int	 vtcon_alloc_scports(struct vtcon_softc *);
161 static int	 vtcon_alloc_virtqueues(struct vtcon_softc *);
162 static void	 vtcon_read_config(struct vtcon_softc *,
163 		     struct virtio_console_config *);
164 
165 static void	 vtcon_determine_max_ports(struct vtcon_softc *,
166 		     struct virtio_console_config *);
167 static void	 vtcon_destroy_ports(struct vtcon_softc *);
168 static void	 vtcon_stop(struct vtcon_softc *);
169 
170 static int	 vtcon_ctrl_event_enqueue(struct vtcon_softc *,
171 		     struct virtio_console_control *);
172 static int	 vtcon_ctrl_event_create(struct vtcon_softc *);
173 static void	 vtcon_ctrl_event_requeue(struct vtcon_softc *,
174 		     struct virtio_console_control *);
175 static int	 vtcon_ctrl_event_populate(struct vtcon_softc *);
176 static void	 vtcon_ctrl_event_drain(struct vtcon_softc *);
177 static int	 vtcon_ctrl_init(struct vtcon_softc *);
178 static void	 vtcon_ctrl_deinit(struct vtcon_softc *);
179 static void	 vtcon_ctrl_port_add_event(struct vtcon_softc *, int);
180 static void	 vtcon_ctrl_port_remove_event(struct vtcon_softc *, int);
181 static void	 vtcon_ctrl_port_console_event(struct vtcon_softc *, int);
182 static void	 vtcon_ctrl_port_open_event(struct vtcon_softc *, int);
183 static void	 vtcon_ctrl_port_name_event(struct vtcon_softc *, int,
184 		     const char *, size_t);
185 static void	 vtcon_ctrl_process_event(struct vtcon_softc *,
186 		     struct virtio_console_control *, void *, size_t);
187 static void	 vtcon_ctrl_task_cb(void *, int);
188 static void	 vtcon_ctrl_event_intr(void *);
189 static void	 vtcon_ctrl_poll(struct vtcon_softc *,
190 		     struct virtio_console_control *control);
191 static void	 vtcon_ctrl_send_control(struct vtcon_softc *, uint32_t,
192 		     uint16_t, uint16_t);
193 
194 static int	 vtcon_port_enqueue_buf(struct vtcon_port *, void *, size_t);
195 static int	 vtcon_port_create_buf(struct vtcon_port *);
196 static void	 vtcon_port_requeue_buf(struct vtcon_port *, void *);
197 static int	 vtcon_port_populate(struct vtcon_port *);
198 static void	 vtcon_port_destroy(struct vtcon_port *);
199 static int	 vtcon_port_create(struct vtcon_softc *, int);
200 static void	 vtcon_port_dev_alias(struct vtcon_port *, const char *,
201 		     size_t);
202 static void	 vtcon_port_drain_bufs(struct virtqueue *);
203 static void	 vtcon_port_drain(struct vtcon_port *);
204 static void	 vtcon_port_teardown(struct vtcon_port *);
205 static void	 vtcon_port_change_size(struct vtcon_port *, uint16_t,
206 		     uint16_t);
207 static void	 vtcon_port_update_console_size(struct vtcon_softc *);
208 static void	 vtcon_port_enable_intr(struct vtcon_port *);
209 static void	 vtcon_port_disable_intr(struct vtcon_port *);
210 static void	 vtcon_port_in(struct vtcon_port *);
211 static void	 vtcon_port_intr(void *);
212 static void	 vtcon_port_out(struct vtcon_port *, void *, int);
213 static void	 vtcon_port_submit_event(struct vtcon_port *, uint16_t,
214 		     uint16_t);
215 
216 static int	 vtcon_tty_open(struct tty *);
217 static void	 vtcon_tty_close(struct tty *);
218 static void	 vtcon_tty_outwakeup(struct tty *);
219 static void	 vtcon_tty_free(void *);
220 
221 static void	 vtcon_get_console_size(struct vtcon_softc *, uint16_t *,
222 		     uint16_t *);
223 
224 static void	 vtcon_enable_interrupts(struct vtcon_softc *);
225 static void	 vtcon_disable_interrupts(struct vtcon_softc *);
226 
227 #define vtcon_modern(_sc) (((_sc)->vtcon_features & VIRTIO_F_VERSION_1) != 0)
228 #define vtcon_htog16(_sc, _val)	virtio_htog16(vtcon_modern(_sc), _val)
229 #define vtcon_htog32(_sc, _val)	virtio_htog32(vtcon_modern(_sc), _val)
230 #define vtcon_htog64(_sc, _val)	virtio_htog64(vtcon_modern(_sc), _val)
231 #define vtcon_gtoh16(_sc, _val)	virtio_gtoh16(vtcon_modern(_sc), _val)
232 #define vtcon_gtoh32(_sc, _val)	virtio_gtoh32(vtcon_modern(_sc), _val)
233 #define vtcon_gtoh64(_sc, _val)	virtio_gtoh64(vtcon_modern(_sc), _val)
234 
235 static int	 vtcon_pending_free;
236 
237 static struct ttydevsw vtcon_tty_class = {
238 	.tsw_flags	= 0,
239 	.tsw_open	= vtcon_tty_open,
240 	.tsw_close	= vtcon_tty_close,
241 	.tsw_outwakeup	= vtcon_tty_outwakeup,
242 	.tsw_free	= vtcon_tty_free,
243 };
244 
245 static device_method_t vtcon_methods[] = {
246 	/* Device methods. */
247 	DEVMETHOD(device_probe,		vtcon_probe),
248 	DEVMETHOD(device_attach,	vtcon_attach),
249 	DEVMETHOD(device_detach,	vtcon_detach),
250 
251 	/* VirtIO methods. */
252 	DEVMETHOD(virtio_config_change,	vtcon_config_change),
253 
254 	DEVMETHOD_END
255 };
256 
257 static driver_t vtcon_driver = {
258 	"vtcon",
259 	vtcon_methods,
260 	sizeof(struct vtcon_softc)
261 };
262 
263 VIRTIO_DRIVER_MODULE(virtio_console, vtcon_driver, vtcon_modevent, NULL);
264 MODULE_VERSION(virtio_console, 1);
265 MODULE_DEPEND(virtio_console, virtio, 1, 1, 1);
266 
267 VIRTIO_SIMPLE_PNPINFO(virtio_console, VIRTIO_ID_CONSOLE,
268     "VirtIO Console Adapter");
269 
270 static int
271 vtcon_modevent(module_t mod, int type, void *unused)
272 {
273 	int error;
274 
275 	switch (type) {
276 	case MOD_LOAD:
277 		error = 0;
278 		break;
279 	case MOD_QUIESCE:
280 		error = 0;
281 		break;
282 	case MOD_UNLOAD:
283 		vtcon_drain_all();
284 		error = 0;
285 		break;
286 	case MOD_SHUTDOWN:
287 		error = 0;
288 		break;
289 	default:
290 		error = EOPNOTSUPP;
291 		break;
292 	}
293 
294 	return (error);
295 }
296 
297 static void
298 vtcon_drain_all(void)
299 {
300 	int first;
301 
302 	for (first = 1; vtcon_pending_free != 0; first = 0) {
303 		if (first != 0) {
304 			printf("virtio_console: Waiting for all detached TTY "
305 			    "devices to have open fds closed.\n");
306 		}
307 		pause("vtcondra", hz);
308 	}
309 }
310 
311 static int
312 vtcon_probe(device_t dev)
313 {
314 	return (VIRTIO_SIMPLE_PROBE(dev, virtio_console));
315 }
316 
317 static int
318 vtcon_attach(device_t dev)
319 {
320 	struct vtcon_softc *sc;
321 	struct virtio_console_config concfg;
322 	int error;
323 
324 	sc = device_get_softc(dev);
325 	sc->vtcon_dev = dev;
326 	virtio_set_feature_desc(dev, vtcon_feature_desc);
327 
328 	mtx_init(&sc->vtcon_mtx, "vtconmtx", NULL, MTX_DEF);
329 	mtx_init(&sc->vtcon_ctrl_tx_mtx, "vtconctrlmtx", NULL, MTX_DEF);
330 
331 	error = vtcon_setup_features(sc);
332 	if (error) {
333 		device_printf(dev, "cannot setup features\n");
334 		goto fail;
335 	}
336 
337 	vtcon_read_config(sc, &concfg);
338 	vtcon_determine_max_ports(sc, &concfg);
339 
340 	error = vtcon_alloc_scports(sc);
341 	if (error) {
342 		device_printf(dev, "cannot allocate softc port structures\n");
343 		goto fail;
344 	}
345 
346 	error = vtcon_alloc_virtqueues(sc);
347 	if (error) {
348 		device_printf(dev, "cannot allocate virtqueues\n");
349 		goto fail;
350 	}
351 
352 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
353 		TASK_INIT(&sc->vtcon_ctrl_task, 0, vtcon_ctrl_task_cb, sc);
354 		error = vtcon_ctrl_init(sc);
355 		if (error)
356 			goto fail;
357 	} else {
358 		error = vtcon_port_create(sc, 0);
359 		if (error)
360 			goto fail;
361 		if (sc->vtcon_flags & VTCON_FLAG_SIZE)
362 			vtcon_port_update_console_size(sc);
363 	}
364 
365 	error = virtio_setup_intr(dev, INTR_TYPE_TTY);
366 	if (error) {
367 		device_printf(dev, "cannot setup virtqueue interrupts\n");
368 		goto fail;
369 	}
370 
371 	vtcon_enable_interrupts(sc);
372 
373 	vtcon_ctrl_send_control(sc, VIRTIO_CONSOLE_BAD_ID,
374 	    VIRTIO_CONSOLE_DEVICE_READY, 1);
375 
376 fail:
377 	if (error)
378 		vtcon_detach(dev);
379 
380 	return (error);
381 }
382 
383 static int
384 vtcon_detach(device_t dev)
385 {
386 	struct vtcon_softc *sc;
387 
388 	sc = device_get_softc(dev);
389 
390 	VTCON_LOCK(sc);
391 	sc->vtcon_flags |= VTCON_FLAG_DETACHED;
392 	if (device_is_attached(dev))
393 		vtcon_stop(sc);
394 	VTCON_UNLOCK(sc);
395 
396 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
397 		taskqueue_drain(taskqueue_thread, &sc->vtcon_ctrl_task);
398 		vtcon_ctrl_deinit(sc);
399 	}
400 
401 	vtcon_destroy_ports(sc);
402 	mtx_destroy(&sc->vtcon_mtx);
403 	mtx_destroy(&sc->vtcon_ctrl_tx_mtx);
404 
405 	return (0);
406 }
407 
408 static int
409 vtcon_config_change(device_t dev)
410 {
411 	struct vtcon_softc *sc;
412 
413 	sc = device_get_softc(dev);
414 
415 	/*
416 	 * When the multiport feature is negotiated, all configuration
417 	 * changes are done through control virtqueue events.
418 	 */
419 	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0) {
420 		if (sc->vtcon_flags & VTCON_FLAG_SIZE)
421 			vtcon_port_update_console_size(sc);
422 	}
423 
424 	return (0);
425 }
426 
427 static int
428 vtcon_negotiate_features(struct vtcon_softc *sc)
429 {
430 	device_t dev;
431 	uint64_t features;
432 
433 	dev = sc->vtcon_dev;
434 	features = VTCON_FEATURES;
435 
436 	sc->vtcon_features = virtio_negotiate_features(dev, features);
437 	return (virtio_finalize_features(dev));
438 }
439 
440 static int
441 vtcon_setup_features(struct vtcon_softc *sc)
442 {
443 	device_t dev;
444 	int error;
445 
446 	dev = sc->vtcon_dev;
447 
448 	error = vtcon_negotiate_features(sc);
449 	if (error)
450 		return (error);
451 
452 	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_SIZE))
453 		sc->vtcon_flags |= VTCON_FLAG_SIZE;
454 	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_MULTIPORT))
455 		sc->vtcon_flags |= VTCON_FLAG_MULTIPORT;
456 
457 	return (0);
458 }
459 
460 #define VTCON_GET_CONFIG(_dev, _feature, _field, _cfg)			\
461 	if (virtio_with_feature(_dev, _feature)) {			\
462 		virtio_read_device_config(_dev,				\
463 		    offsetof(struct virtio_console_config, _field),	\
464 		    &(_cfg)->_field, sizeof((_cfg)->_field));		\
465 	}
466 
467 static void
468 vtcon_read_config(struct vtcon_softc *sc, struct virtio_console_config *concfg)
469 {
470 	device_t dev;
471 
472 	dev = sc->vtcon_dev;
473 
474 	bzero(concfg, sizeof(struct virtio_console_config));
475 
476 	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, cols, concfg);
477 	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, rows, concfg);
478 	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_MULTIPORT, max_nr_ports, concfg);
479 }
480 
481 #undef VTCON_GET_CONFIG
482 
483 static int
484 vtcon_alloc_scports(struct vtcon_softc *sc)
485 {
486 	struct vtcon_softc_port *scport;
487 	int max, i;
488 
489 	max = sc->vtcon_max_ports;
490 
491 	sc->vtcon_ports = malloc(sizeof(struct vtcon_softc_port) * max,
492 	    M_DEVBUF, M_NOWAIT | M_ZERO);
493 	if (sc->vtcon_ports == NULL)
494 		return (ENOMEM);
495 
496 	for (i = 0; i < max; i++) {
497 		scport = &sc->vtcon_ports[i];
498 		scport->vcsp_sc = sc;
499 	}
500 
501 	return (0);
502 }
503 
504 static int
505 vtcon_alloc_virtqueues(struct vtcon_softc *sc)
506 {
507 	device_t dev;
508 	struct vq_alloc_info *info;
509 	struct vtcon_softc_port *scport;
510 	int i, idx, portidx, nvqs, error;
511 
512 	dev = sc->vtcon_dev;
513 
514 	nvqs = sc->vtcon_max_ports * 2;
515 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
516 		nvqs += 2;
517 
518 	info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT);
519 	if (info == NULL)
520 		return (ENOMEM);
521 
522 	for (i = 0, idx = 0, portidx = 0; i < nvqs / 2; i++, idx += 2) {
523 		if (i == 1) {
524 			/* The control virtqueues are after the first port. */
525 			VQ_ALLOC_INFO_INIT(&info[idx], 0,
526 			    vtcon_ctrl_event_intr, sc, &sc->vtcon_ctrl_rxvq,
527 			    "%s-control rx", device_get_nameunit(dev));
528 			VQ_ALLOC_INFO_INIT(&info[idx+1], 0,
529 			    NULL, sc, &sc->vtcon_ctrl_txvq,
530 			    "%s-control tx", device_get_nameunit(dev));
531 			continue;
532 		}
533 
534 		scport = &sc->vtcon_ports[portidx];
535 
536 		VQ_ALLOC_INFO_INIT(&info[idx], 0, vtcon_port_intr,
537 		    scport, &scport->vcsp_invq, "%s-port%d in",
538 		    device_get_nameunit(dev), i);
539 		VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL,
540 		    NULL, &scport->vcsp_outvq, "%s-port%d out",
541 		    device_get_nameunit(dev), i);
542 
543 		portidx++;
544 	}
545 
546 	error = virtio_alloc_virtqueues(dev, nvqs, info);
547 	free(info, M_TEMP);
548 
549 	return (error);
550 }
551 
552 static void
553 vtcon_determine_max_ports(struct vtcon_softc *sc,
554     struct virtio_console_config *concfg)
555 {
556 
557 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
558 		sc->vtcon_max_ports =
559 		    min(concfg->max_nr_ports, VTCON_MAX_PORTS);
560 		if (sc->vtcon_max_ports == 0)
561 			sc->vtcon_max_ports = 1;
562 	} else
563 		sc->vtcon_max_ports = 1;
564 }
565 
566 static void
567 vtcon_destroy_ports(struct vtcon_softc *sc)
568 {
569 	struct vtcon_softc_port *scport;
570 	struct vtcon_port *port;
571 	struct virtqueue *vq;
572 	int i;
573 
574 	if (sc->vtcon_ports == NULL)
575 		return;
576 
577 	VTCON_LOCK(sc);
578 	for (i = 0; i < sc->vtcon_max_ports; i++) {
579 		scport = &sc->vtcon_ports[i];
580 
581 		port = scport->vcsp_port;
582 		if (port != NULL) {
583 			scport->vcsp_port = NULL;
584 			VTCON_PORT_LOCK(port);
585 			VTCON_UNLOCK(sc);
586 			vtcon_port_teardown(port);
587 			VTCON_LOCK(sc);
588 		}
589 
590 		vq = scport->vcsp_invq;
591 		if (vq != NULL)
592 			vtcon_port_drain_bufs(vq);
593 	}
594 	VTCON_UNLOCK(sc);
595 
596 	free(sc->vtcon_ports, M_DEVBUF);
597 	sc->vtcon_ports = NULL;
598 }
599 
600 static void
601 vtcon_stop(struct vtcon_softc *sc)
602 {
603 
604 	vtcon_disable_interrupts(sc);
605 	virtio_stop(sc->vtcon_dev);
606 }
607 
608 static int
609 vtcon_ctrl_event_enqueue(struct vtcon_softc *sc,
610     struct virtio_console_control *control)
611 {
612 	struct sglist_seg segs[2];
613 	struct sglist sg;
614 	struct virtqueue *vq;
615 	int error __diagused;
616 
617 	vq = sc->vtcon_ctrl_rxvq;
618 
619 	sglist_init(&sg, 2, segs);
620 	error = sglist_append(&sg, control, VTCON_CTRL_BUFSZ);
621 	KASSERT(error == 0, ("%s: error %d adding control to sglist",
622 	    __func__, error));
623 
624 	return (virtqueue_enqueue(vq, control, &sg, 0, sg.sg_nseg));
625 }
626 
627 static int
628 vtcon_ctrl_event_create(struct vtcon_softc *sc)
629 {
630 	struct virtio_console_control *control;
631 	int error;
632 
633 	control = malloc(VTCON_CTRL_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT);
634 	if (control == NULL)
635 		return (ENOMEM);
636 
637 	error = vtcon_ctrl_event_enqueue(sc, control);
638 	if (error)
639 		free(control, M_DEVBUF);
640 
641 	return (error);
642 }
643 
644 static void
645 vtcon_ctrl_event_requeue(struct vtcon_softc *sc,
646     struct virtio_console_control *control)
647 {
648 	int error __diagused;
649 
650 	bzero(control, VTCON_CTRL_BUFSZ);
651 
652 	error = vtcon_ctrl_event_enqueue(sc, control);
653 	KASSERT(error == 0,
654 	    ("%s: cannot requeue control buffer %d", __func__, error));
655 }
656 
657 static int
658 vtcon_ctrl_event_populate(struct vtcon_softc *sc)
659 {
660 	struct virtqueue *vq;
661 	int nbufs, error;
662 
663 	vq = sc->vtcon_ctrl_rxvq;
664 	error = ENOSPC;
665 
666 	for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
667 		error = vtcon_ctrl_event_create(sc);
668 		if (error)
669 			break;
670 	}
671 
672 	if (nbufs > 0) {
673 		virtqueue_notify(vq);
674 		error = 0;
675 	}
676 
677 	return (error);
678 }
679 
680 static void
681 vtcon_ctrl_event_drain(struct vtcon_softc *sc)
682 {
683 	struct virtio_console_control *control;
684 	struct virtqueue *vq;
685 	int last;
686 
687 	vq = sc->vtcon_ctrl_rxvq;
688 	last = 0;
689 
690 	if (vq == NULL)
691 		return;
692 
693 	VTCON_LOCK(sc);
694 	while ((control = virtqueue_drain(vq, &last)) != NULL)
695 		free(control, M_DEVBUF);
696 	VTCON_UNLOCK(sc);
697 }
698 
699 static int
700 vtcon_ctrl_init(struct vtcon_softc *sc)
701 {
702 	int error;
703 
704 	error = vtcon_ctrl_event_populate(sc);
705 
706 	return (error);
707 }
708 
709 static void
710 vtcon_ctrl_deinit(struct vtcon_softc *sc)
711 {
712 
713 	vtcon_ctrl_event_drain(sc);
714 }
715 
716 static void
717 vtcon_ctrl_port_add_event(struct vtcon_softc *sc, int id)
718 {
719 	device_t dev;
720 	int error;
721 
722 	dev = sc->vtcon_dev;
723 
724 	/* This single thread only way for ports to be created. */
725 	if (sc->vtcon_ports[id].vcsp_port != NULL) {
726 		device_printf(dev, "%s: adding port %d, but already exists\n",
727 		    __func__, id);
728 		return;
729 	}
730 
731 	error = vtcon_port_create(sc, id);
732 	if (error) {
733 		device_printf(dev, "%s: cannot create port %d: %d\n",
734 		    __func__, id, error);
735 		vtcon_ctrl_send_control(sc, id, VIRTIO_CONSOLE_PORT_READY, 0);
736 		return;
737 	}
738 }
739 
740 static void
741 vtcon_ctrl_port_remove_event(struct vtcon_softc *sc, int id)
742 {
743 	device_t dev;
744 	struct vtcon_softc_port *scport;
745 	struct vtcon_port *port;
746 
747 	dev = sc->vtcon_dev;
748 	scport = &sc->vtcon_ports[id];
749 
750 	VTCON_LOCK(sc);
751 	port = scport->vcsp_port;
752 	if (port == NULL) {
753 		VTCON_UNLOCK(sc);
754 		device_printf(dev, "%s: remove port %d, but does not exist\n",
755 		    __func__, id);
756 		return;
757 	}
758 
759 	scport->vcsp_port = NULL;
760 	VTCON_PORT_LOCK(port);
761 	VTCON_UNLOCK(sc);
762 	vtcon_port_teardown(port);
763 }
764 
765 static void
766 vtcon_ctrl_port_console_event(struct vtcon_softc *sc, int id)
767 {
768 	device_t dev;
769 	struct vtcon_softc_port *scport;
770 	struct vtcon_port *port;
771 
772 	dev = sc->vtcon_dev;
773 	scport = &sc->vtcon_ports[id];
774 
775 	VTCON_LOCK(sc);
776 	port = scport->vcsp_port;
777 	if (port == NULL) {
778 		VTCON_UNLOCK(sc);
779 		device_printf(dev, "%s: console port %d, but does not exist\n",
780 		    __func__, id);
781 		return;
782 	}
783 
784 	VTCON_PORT_LOCK(port);
785 	VTCON_UNLOCK(sc);
786 	port->vtcport_flags |= VTCON_PORT_FLAG_CONSOLE;
787 	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
788 	VTCON_PORT_UNLOCK(port);
789 }
790 
791 static void
792 vtcon_ctrl_port_open_event(struct vtcon_softc *sc, int id)
793 {
794 	device_t dev;
795 	struct vtcon_softc_port *scport;
796 	struct vtcon_port *port;
797 
798 	dev = sc->vtcon_dev;
799 	scport = &sc->vtcon_ports[id];
800 
801 	VTCON_LOCK(sc);
802 	port = scport->vcsp_port;
803 	if (port == NULL) {
804 		VTCON_UNLOCK(sc);
805 		device_printf(dev, "%s: open port %d, but does not exist\n",
806 		    __func__, id);
807 		return;
808 	}
809 
810 	VTCON_PORT_LOCK(port);
811 	VTCON_UNLOCK(sc);
812 	vtcon_port_enable_intr(port);
813 	VTCON_PORT_UNLOCK(port);
814 }
815 
816 static void
817 vtcon_ctrl_port_name_event(struct vtcon_softc *sc, int id, const char *name,
818     size_t len)
819 {
820 	device_t dev;
821 	struct vtcon_softc_port *scport;
822 	struct vtcon_port *port;
823 
824 	dev = sc->vtcon_dev;
825 	scport = &sc->vtcon_ports[id];
826 
827 	/*
828 	 * The VirtIO specification says the NUL terminator is not included in
829 	 * the length, but QEMU includes it. Adjust the length if needed.
830 	 */
831 	if (name == NULL || len == 0)
832 		return;
833 	if (name[len - 1] == '\0') {
834 		len--;
835 		if (len == 0)
836 			return;
837 	}
838 
839 	VTCON_LOCK(sc);
840 	port = scport->vcsp_port;
841 	if (port == NULL) {
842 		VTCON_UNLOCK(sc);
843 		device_printf(dev, "%s: name port %d, but does not exist\n",
844 		    __func__, id);
845 		return;
846 	}
847 
848 	VTCON_PORT_LOCK(port);
849 	VTCON_UNLOCK(sc);
850 	vtcon_port_dev_alias(port, name, len);
851 	VTCON_PORT_UNLOCK(port);
852 }
853 
854 static void
855 vtcon_ctrl_process_event(struct vtcon_softc *sc,
856     struct virtio_console_control *control, void *data, size_t data_len)
857 {
858 	device_t dev;
859 	uint32_t id;
860 	uint16_t event;
861 
862 	dev = sc->vtcon_dev;
863 	id = vtcon_htog32(sc, control->id);
864 	event = vtcon_htog16(sc, control->event);
865 
866 	if (id >= sc->vtcon_max_ports) {
867 		device_printf(dev, "%s: event %d invalid port ID %d\n",
868 		    __func__, event, id);
869 		return;
870 	}
871 
872 	switch (event) {
873 	case VIRTIO_CONSOLE_PORT_ADD:
874 		vtcon_ctrl_port_add_event(sc, id);
875 		break;
876 
877 	case VIRTIO_CONSOLE_PORT_REMOVE:
878 		vtcon_ctrl_port_remove_event(sc, id);
879 		break;
880 
881 	case VIRTIO_CONSOLE_CONSOLE_PORT:
882 		vtcon_ctrl_port_console_event(sc, id);
883 		break;
884 
885 	case VIRTIO_CONSOLE_RESIZE:
886 		break;
887 
888 	case VIRTIO_CONSOLE_PORT_OPEN:
889 		vtcon_ctrl_port_open_event(sc, id);
890 		break;
891 
892 	case VIRTIO_CONSOLE_PORT_NAME:
893 		vtcon_ctrl_port_name_event(sc, id, (const char *)data, data_len);
894 		break;
895 	}
896 }
897 
898 static void
899 vtcon_ctrl_task_cb(void *xsc, int pending)
900 {
901 	struct vtcon_softc *sc;
902 	struct virtqueue *vq;
903 	struct virtio_console_control *control;
904 	void *data;
905 	size_t data_len;
906 	int detached;
907 	uint32_t len;
908 
909 	sc = xsc;
910 	vq = sc->vtcon_ctrl_rxvq;
911 
912 	VTCON_LOCK(sc);
913 
914 	while ((detached = (sc->vtcon_flags & VTCON_FLAG_DETACHED)) == 0) {
915 		control = virtqueue_dequeue(vq, &len);
916 		if (control == NULL)
917 			break;
918 
919 		if (len > sizeof(struct virtio_console_control)) {
920 			data = (void *) &control[1];
921 			data_len = len - sizeof(struct virtio_console_control);
922 		} else {
923 			data = NULL;
924 			data_len = 0;
925 		}
926 
927 		VTCON_UNLOCK(sc);
928 		vtcon_ctrl_process_event(sc, control, data, data_len);
929 		VTCON_LOCK(sc);
930 		vtcon_ctrl_event_requeue(sc, control);
931 	}
932 
933 	if (!detached) {
934 		virtqueue_notify(vq);
935 		if (virtqueue_enable_intr(vq) != 0)
936 			taskqueue_enqueue(taskqueue_thread,
937 			    &sc->vtcon_ctrl_task);
938 	}
939 
940 	VTCON_UNLOCK(sc);
941 }
942 
943 static void
944 vtcon_ctrl_event_intr(void *xsc)
945 {
946 	struct vtcon_softc *sc;
947 
948 	sc = xsc;
949 
950 	/*
951 	 * Only some events require us to potentially block, but it
952 	 * easier to just defer all event handling to the taskqueue.
953 	 */
954 	taskqueue_enqueue(taskqueue_thread, &sc->vtcon_ctrl_task);
955 }
956 
957 static void
958 vtcon_ctrl_poll(struct vtcon_softc *sc,
959     struct virtio_console_control *control)
960 {
961 	struct sglist_seg segs[2];
962 	struct sglist sg;
963 	struct virtqueue *vq;
964 	int error;
965 
966 	vq = sc->vtcon_ctrl_txvq;
967 
968 	sglist_init(&sg, 2, segs);
969 	error = sglist_append(&sg, control,
970 	    sizeof(struct virtio_console_control));
971 	KASSERT(error == 0, ("%s: error %d adding control to sglist",
972 	    __func__, error));
973 
974 	/*
975 	 * We cannot use the softc lock to serialize access to this
976 	 * virtqueue since this is called from the tty layer with the
977 	 * port lock held. Acquiring the softc would violate our lock
978 	 * ordering.
979 	 */
980 	VTCON_CTRL_TX_LOCK(sc);
981 	KASSERT(virtqueue_empty(vq),
982 	    ("%s: virtqueue is not emtpy", __func__));
983 	error = virtqueue_enqueue(vq, control, &sg, sg.sg_nseg, 0);
984 	if (error == 0) {
985 		virtqueue_notify(vq);
986 		virtqueue_poll(vq, NULL);
987 	}
988 	VTCON_CTRL_TX_UNLOCK(sc);
989 }
990 
991 static void
992 vtcon_ctrl_send_control(struct vtcon_softc *sc, uint32_t portid,
993     uint16_t event, uint16_t value)
994 {
995 	struct virtio_console_control control;
996 
997 	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0)
998 		return;
999 
1000 	control.id = vtcon_gtoh32(sc, portid);
1001 	control.event = vtcon_gtoh16(sc, event);
1002 	control.value = vtcon_gtoh16(sc, value);
1003 
1004 	vtcon_ctrl_poll(sc, &control);
1005 }
1006 
1007 static int
1008 vtcon_port_enqueue_buf(struct vtcon_port *port, void *buf, size_t len)
1009 {
1010 	struct sglist_seg segs[2];
1011 	struct sglist sg;
1012 	struct virtqueue *vq;
1013 	int error;
1014 
1015 	vq = port->vtcport_invq;
1016 
1017 	sglist_init(&sg, 2, segs);
1018 	error = sglist_append(&sg, buf, len);
1019 	KASSERT(error == 0,
1020 	    ("%s: error %d adding buffer to sglist", __func__, error));
1021 
1022 	error = virtqueue_enqueue(vq, buf, &sg, 0, sg.sg_nseg);
1023 
1024 	return (error);
1025 }
1026 
1027 static int
1028 vtcon_port_create_buf(struct vtcon_port *port)
1029 {
1030 	void *buf;
1031 	int error;
1032 
1033 	buf = malloc(VTCON_BULK_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT);
1034 	if (buf == NULL)
1035 		return (ENOMEM);
1036 
1037 	error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ);
1038 	if (error)
1039 		free(buf, M_DEVBUF);
1040 
1041 	return (error);
1042 }
1043 
1044 static void
1045 vtcon_port_requeue_buf(struct vtcon_port *port, void *buf)
1046 {
1047 	int error __diagused;
1048 
1049 	error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ);
1050 	KASSERT(error == 0,
1051 	    ("%s: cannot requeue input buffer %d", __func__, error));
1052 }
1053 
1054 static int
1055 vtcon_port_populate(struct vtcon_port *port)
1056 {
1057 	struct virtqueue *vq;
1058 	int nbufs, error;
1059 
1060 	vq = port->vtcport_invq;
1061 	error = ENOSPC;
1062 
1063 	for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
1064 		error = vtcon_port_create_buf(port);
1065 		if (error)
1066 			break;
1067 	}
1068 
1069 	if (nbufs > 0) {
1070 		virtqueue_notify(vq);
1071 		error = 0;
1072 	}
1073 
1074 	return (error);
1075 }
1076 
1077 static void
1078 vtcon_port_destroy(struct vtcon_port *port)
1079 {
1080 
1081 	port->vtcport_sc = NULL;
1082 	port->vtcport_scport = NULL;
1083 	port->vtcport_invq = NULL;
1084 	port->vtcport_outvq = NULL;
1085 	port->vtcport_id = -1;
1086 	mtx_destroy(&port->vtcport_mtx);
1087 	free(port, M_DEVBUF);
1088 }
1089 
1090 static int
1091 vtcon_port_init_vqs(struct vtcon_port *port)
1092 {
1093 	struct vtcon_softc_port *scport;
1094 	int error;
1095 
1096 	scport = port->vtcport_scport;
1097 
1098 	port->vtcport_invq = scport->vcsp_invq;
1099 	port->vtcport_outvq = scport->vcsp_outvq;
1100 
1101 	/*
1102 	 * Free any data left over from when this virtqueue was in use by a
1103 	 * prior port. We have not yet notified the host that the port is
1104 	 * ready, so assume nothing in the virtqueue can be for us.
1105 	 */
1106 	vtcon_port_drain(port);
1107 
1108 	KASSERT(virtqueue_empty(port->vtcport_invq),
1109 	    ("%s: in virtqueue is not empty", __func__));
1110 	KASSERT(virtqueue_empty(port->vtcport_outvq),
1111 	    ("%s: out virtqueue is not empty", __func__));
1112 
1113 	error = vtcon_port_populate(port);
1114 	if (error)
1115 		return (error);
1116 
1117 	return (0);
1118 }
1119 
1120 static int
1121 vtcon_port_create(struct vtcon_softc *sc, int id)
1122 {
1123 	device_t dev;
1124 	struct vtcon_softc_port *scport;
1125 	struct vtcon_port *port;
1126 	int error;
1127 
1128 	dev = sc->vtcon_dev;
1129 	scport = &sc->vtcon_ports[id];
1130 
1131 	VTCON_ASSERT_VALID_PORTID(sc, id);
1132 	MPASS(scport->vcsp_port == NULL);
1133 
1134 	port = malloc(sizeof(struct vtcon_port), M_DEVBUF, M_NOWAIT | M_ZERO);
1135 	if (port == NULL)
1136 		return (ENOMEM);
1137 
1138 	port->vtcport_sc = sc;
1139 	port->vtcport_scport = scport;
1140 	port->vtcport_id = id;
1141 	mtx_init(&port->vtcport_mtx, "vtcpmtx", NULL, MTX_DEF);
1142 	port->vtcport_tty = tty_alloc_mutex(&vtcon_tty_class, port,
1143 	    &port->vtcport_mtx);
1144 
1145 	error = vtcon_port_init_vqs(port);
1146 	if (error) {
1147 		VTCON_PORT_LOCK(port);
1148 		vtcon_port_teardown(port);
1149 		return (error);
1150 	}
1151 
1152 	VTCON_LOCK(sc);
1153 	VTCON_PORT_LOCK(port);
1154 	scport->vcsp_port = port;
1155 	vtcon_port_enable_intr(port);
1156 	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_READY, 1);
1157 	VTCON_PORT_UNLOCK(port);
1158 	VTCON_UNLOCK(sc);
1159 
1160 	tty_makedev(port->vtcport_tty, NULL, "%s%r.%r", VTCON_TTY_PREFIX,
1161 	    device_get_unit(dev), id);
1162 
1163 	return (0);
1164 }
1165 
1166 static void
1167 vtcon_port_dev_alias(struct vtcon_port *port, const char *name, size_t len)
1168 {
1169 	struct vtcon_softc *sc;
1170 	struct cdev *pdev;
1171 	struct tty *tp;
1172 	int i, error;
1173 
1174 	sc = port->vtcport_sc;
1175 	tp = port->vtcport_tty;
1176 
1177 	if (port->vtcport_flags & VTCON_PORT_FLAG_ALIAS)
1178 		return;
1179 
1180 	/* Port name is UTF-8, but we can only handle ASCII. */
1181 	for (i = 0; i < len; i++) {
1182 		if (!isascii(name[i]))
1183 			return;
1184 	}
1185 
1186 	/*
1187 	 * Port name may not conform to the devfs requirements so we cannot use
1188 	 * tty_makealias() because the MAKEDEV_CHECKNAME flag must be specified.
1189 	 */
1190 	error = make_dev_alias_p(MAKEDEV_NOWAIT | MAKEDEV_CHECKNAME, &pdev,
1191 	    tp->t_dev, "%s/%*s", VTCON_TTY_ALIAS_PREFIX, (int)len, name);
1192 	if (error) {
1193 		device_printf(sc->vtcon_dev,
1194 		    "%s: cannot make dev alias (%s/%*s) error %d\n", __func__,
1195 		    VTCON_TTY_ALIAS_PREFIX, (int)len, name, error);
1196 	} else
1197 		port->vtcport_flags |= VTCON_PORT_FLAG_ALIAS;
1198 }
1199 
1200 static void
1201 vtcon_port_drain_bufs(struct virtqueue *vq)
1202 {
1203 	void *buf;
1204 	int last;
1205 
1206 	last = 0;
1207 
1208 	while ((buf = virtqueue_drain(vq, &last)) != NULL)
1209 		free(buf, M_DEVBUF);
1210 }
1211 
1212 static void
1213 vtcon_port_drain(struct vtcon_port *port)
1214 {
1215 
1216 	vtcon_port_drain_bufs(port->vtcport_invq);
1217 }
1218 
1219 static void
1220 vtcon_port_teardown(struct vtcon_port *port)
1221 {
1222 	struct tty *tp;
1223 
1224 	tp = port->vtcport_tty;
1225 
1226 	port->vtcport_flags |= VTCON_PORT_FLAG_GONE;
1227 
1228 	if (tp != NULL) {
1229 		atomic_add_int(&vtcon_pending_free, 1);
1230 		tty_rel_gone(tp);
1231 	} else
1232 		vtcon_port_destroy(port);
1233 }
1234 
1235 static void
1236 vtcon_port_change_size(struct vtcon_port *port, uint16_t cols, uint16_t rows)
1237 {
1238 	struct tty *tp;
1239 	struct winsize sz;
1240 
1241 	tp = port->vtcport_tty;
1242 
1243 	if (tp == NULL)
1244 		return;
1245 
1246 	bzero(&sz, sizeof(struct winsize));
1247 	sz.ws_col = cols;
1248 	sz.ws_row = rows;
1249 
1250 	tty_set_winsize(tp, &sz);
1251 }
1252 
1253 static void
1254 vtcon_port_update_console_size(struct vtcon_softc *sc)
1255 {
1256 	struct vtcon_port *port;
1257 	struct vtcon_softc_port *scport;
1258 	uint16_t cols, rows;
1259 
1260 	vtcon_get_console_size(sc, &cols, &rows);
1261 
1262 	/*
1263 	 * For now, assume the first (only) port is the console. Note
1264 	 * QEMU does not implement this feature yet.
1265 	 */
1266 	scport = &sc->vtcon_ports[0];
1267 
1268 	VTCON_LOCK(sc);
1269 	port = scport->vcsp_port;
1270 
1271 	if (port != NULL) {
1272 		VTCON_PORT_LOCK(port);
1273 		VTCON_UNLOCK(sc);
1274 		vtcon_port_change_size(port, cols, rows);
1275 		VTCON_PORT_UNLOCK(port);
1276 	} else
1277 		VTCON_UNLOCK(sc);
1278 }
1279 
1280 static void
1281 vtcon_port_enable_intr(struct vtcon_port *port)
1282 {
1283 
1284 	/*
1285 	 * NOTE: The out virtqueue is always polled, so its interrupt
1286 	 * kept disabled.
1287 	 */
1288 	virtqueue_enable_intr(port->vtcport_invq);
1289 }
1290 
1291 static void
1292 vtcon_port_disable_intr(struct vtcon_port *port)
1293 {
1294 
1295 	if (port->vtcport_invq != NULL)
1296 		virtqueue_disable_intr(port->vtcport_invq);
1297 	if (port->vtcport_outvq != NULL)
1298 		virtqueue_disable_intr(port->vtcport_outvq);
1299 }
1300 
1301 static void
1302 vtcon_port_in(struct vtcon_port *port)
1303 {
1304 	struct virtqueue *vq;
1305 	struct tty *tp;
1306 	char *buf;
1307 	uint32_t len;
1308 	int i, deq;
1309 
1310 	tp = port->vtcport_tty;
1311 	vq = port->vtcport_invq;
1312 
1313 again:
1314 	deq = 0;
1315 
1316 	while ((buf = virtqueue_dequeue(vq, &len)) != NULL) {
1317 		for (i = 0; i < len; i++) {
1318 #if defined(KDB)
1319 			if (port->vtcport_flags & VTCON_PORT_FLAG_CONSOLE)
1320 				kdb_alt_break(buf[i],
1321 				    &port->vtcport_alt_break_state);
1322 #endif
1323 			ttydisc_rint(tp, buf[i], 0);
1324 		}
1325 		vtcon_port_requeue_buf(port, buf);
1326 		deq++;
1327 	}
1328 	ttydisc_rint_done(tp);
1329 
1330 	if (deq > 0)
1331 		virtqueue_notify(vq);
1332 
1333 	if (virtqueue_enable_intr(vq) != 0)
1334 		goto again;
1335 }
1336 
1337 static void
1338 vtcon_port_intr(void *scportx)
1339 {
1340 	struct vtcon_softc_port *scport;
1341 	struct vtcon_softc *sc;
1342 	struct vtcon_port *port;
1343 
1344 	scport = scportx;
1345 	sc = scport->vcsp_sc;
1346 
1347 	VTCON_LOCK(sc);
1348 	port = scport->vcsp_port;
1349 	if (port == NULL) {
1350 		VTCON_UNLOCK(sc);
1351 		return;
1352 	}
1353 	VTCON_PORT_LOCK(port);
1354 	VTCON_UNLOCK(sc);
1355 	if ((port->vtcport_flags & VTCON_PORT_FLAG_GONE) == 0)
1356 		vtcon_port_in(port);
1357 	VTCON_PORT_UNLOCK(port);
1358 }
1359 
1360 static void
1361 vtcon_port_out(struct vtcon_port *port, void *buf, int bufsize)
1362 {
1363 	struct sglist_seg segs[2];
1364 	struct sglist sg;
1365 	struct virtqueue *vq;
1366 	int error;
1367 
1368 	vq = port->vtcport_outvq;
1369 	KASSERT(virtqueue_empty(vq),
1370 	    ("%s: port %p out virtqueue not emtpy", __func__, port));
1371 
1372 	sglist_init(&sg, 2, segs);
1373 	error = sglist_append(&sg, buf, bufsize);
1374 	KASSERT(error == 0, ("%s: error %d adding buffer to sglist",
1375 	    __func__, error));
1376 
1377 	error = virtqueue_enqueue(vq, buf, &sg, sg.sg_nseg, 0);
1378 	if (error == 0) {
1379 		virtqueue_notify(vq);
1380 		virtqueue_poll(vq, NULL);
1381 	}
1382 }
1383 
1384 static void
1385 vtcon_port_submit_event(struct vtcon_port *port, uint16_t event,
1386     uint16_t value)
1387 {
1388 	struct vtcon_softc *sc;
1389 
1390 	sc = port->vtcport_sc;
1391 
1392 	vtcon_ctrl_send_control(sc, port->vtcport_id, event, value);
1393 }
1394 
1395 static int
1396 vtcon_tty_open(struct tty *tp)
1397 {
1398 	struct vtcon_port *port;
1399 
1400 	port = tty_softc(tp);
1401 
1402 	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
1403 		return (ENXIO);
1404 
1405 	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
1406 
1407 	return (0);
1408 }
1409 
1410 static void
1411 vtcon_tty_close(struct tty *tp)
1412 {
1413 	struct vtcon_port *port;
1414 
1415 	port = tty_softc(tp);
1416 
1417 	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
1418 		return;
1419 
1420 	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 0);
1421 }
1422 
1423 static void
1424 vtcon_tty_outwakeup(struct tty *tp)
1425 {
1426 	struct vtcon_port *port;
1427 	char buf[VTCON_BULK_BUFSZ];
1428 	int len;
1429 
1430 	port = tty_softc(tp);
1431 
1432 	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
1433 		return;
1434 
1435 	while ((len = ttydisc_getc(tp, buf, sizeof(buf))) != 0)
1436 		vtcon_port_out(port, buf, len);
1437 }
1438 
1439 static void
1440 vtcon_tty_free(void *xport)
1441 {
1442 	struct vtcon_port *port;
1443 
1444 	port = xport;
1445 
1446 	vtcon_port_destroy(port);
1447 	atomic_subtract_int(&vtcon_pending_free, 1);
1448 }
1449 
1450 static void
1451 vtcon_get_console_size(struct vtcon_softc *sc, uint16_t *cols, uint16_t *rows)
1452 {
1453 	struct virtio_console_config concfg;
1454 
1455 	KASSERT(sc->vtcon_flags & VTCON_FLAG_SIZE,
1456 	    ("%s: size feature not negotiated", __func__));
1457 
1458 	vtcon_read_config(sc, &concfg);
1459 
1460 	*cols = concfg.cols;
1461 	*rows = concfg.rows;
1462 }
1463 
1464 static void
1465 vtcon_enable_interrupts(struct vtcon_softc *sc)
1466 {
1467 	struct vtcon_softc_port *scport;
1468 	struct vtcon_port *port;
1469 	int i;
1470 
1471 	VTCON_LOCK(sc);
1472 
1473 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
1474 		virtqueue_enable_intr(sc->vtcon_ctrl_rxvq);
1475 
1476 	for (i = 0; i < sc->vtcon_max_ports; i++) {
1477 		scport = &sc->vtcon_ports[i];
1478 
1479 		port = scport->vcsp_port;
1480 		if (port == NULL)
1481 			continue;
1482 
1483 		VTCON_PORT_LOCK(port);
1484 		vtcon_port_enable_intr(port);
1485 		VTCON_PORT_UNLOCK(port);
1486 	}
1487 
1488 	VTCON_UNLOCK(sc);
1489 }
1490 
1491 static void
1492 vtcon_disable_interrupts(struct vtcon_softc *sc)
1493 {
1494 	struct vtcon_softc_port *scport;
1495 	struct vtcon_port *port;
1496 	int i;
1497 
1498 	VTCON_LOCK_ASSERT(sc);
1499 
1500 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
1501 		virtqueue_disable_intr(sc->vtcon_ctrl_rxvq);
1502 
1503 	for (i = 0; i < sc->vtcon_max_ports; i++) {
1504 		scport = &sc->vtcon_ports[i];
1505 
1506 		port = scport->vcsp_port;
1507 		if (port == NULL)
1508 			continue;
1509 
1510 		VTCON_PORT_LOCK(port);
1511 		vtcon_port_disable_intr(port);
1512 		VTCON_PORT_UNLOCK(port);
1513 	}
1514 }
1515