xref: /freebsd/sys/dev/virtio/console/virtio_console.c (revision 9768746ba83efa02837c5b9c66348db6e900208f)
1 /*-
2  * Copyright (c) 2014, Bryan Venteicher <bryanv@FreeBSD.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice unmodified, this list of conditions, and the following
10  *    disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 /* Driver for VirtIO console devices. */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/ctype.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/malloc.h>
37 #include <sys/module.h>
38 #include <sys/kdb.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/sglist.h>
42 #include <sys/sysctl.h>
43 #include <sys/taskqueue.h>
44 #include <sys/queue.h>
45 
46 #include <sys/conf.h>
47 #include <sys/cons.h>
48 #include <sys/tty.h>
49 
50 #include <machine/bus.h>
51 #include <machine/resource.h>
52 #include <sys/bus.h>
53 
54 #include <dev/virtio/virtio.h>
55 #include <dev/virtio/virtqueue.h>
56 #include <dev/virtio/console/virtio_console.h>
57 
58 #include "virtio_if.h"
59 
60 #define VTCON_MAX_PORTS 32
61 #define VTCON_TTY_PREFIX "V"
62 #define VTCON_TTY_ALIAS_PREFIX "vtcon"
63 #define VTCON_BULK_BUFSZ 128
64 #define VTCON_CTRL_BUFSZ 128
65 
66 /*
67  * The buffers cannot cross more than one page boundary due to the
68  * size of the sglist segment array used.
69  */
70 CTASSERT(VTCON_BULK_BUFSZ <= PAGE_SIZE);
71 CTASSERT(VTCON_CTRL_BUFSZ <= PAGE_SIZE);
72 
73 CTASSERT(sizeof(struct virtio_console_config) <= VTCON_CTRL_BUFSZ);
74 
75 struct vtcon_softc;
76 struct vtcon_softc_port;
77 
78 struct vtcon_port {
79 	struct mtx			 vtcport_mtx;
80 	struct vtcon_softc		*vtcport_sc;
81 	struct vtcon_softc_port		*vtcport_scport;
82 	struct tty			*vtcport_tty;
83 	struct virtqueue		*vtcport_invq;
84 	struct virtqueue		*vtcport_outvq;
85 	int				 vtcport_id;
86 	int				 vtcport_flags;
87 #define VTCON_PORT_FLAG_GONE	0x01
88 #define VTCON_PORT_FLAG_CONSOLE	0x02
89 #define VTCON_PORT_FLAG_ALIAS	0x04
90 
91 #if defined(KDB)
92 	int				 vtcport_alt_break_state;
93 #endif
94 };
95 
96 #define VTCON_PORT_LOCK(_port)		mtx_lock(&(_port)->vtcport_mtx)
97 #define VTCON_PORT_UNLOCK(_port)	mtx_unlock(&(_port)->vtcport_mtx)
98 
99 struct vtcon_softc_port {
100 	struct vtcon_softc	*vcsp_sc;
101 	struct vtcon_port	*vcsp_port;
102 	struct virtqueue	*vcsp_invq;
103 	struct virtqueue	*vcsp_outvq;
104 };
105 
106 struct vtcon_softc {
107 	device_t		 vtcon_dev;
108 	struct mtx		 vtcon_mtx;
109 	uint64_t		 vtcon_features;
110 	uint32_t		 vtcon_max_ports;
111 	uint32_t		 vtcon_flags;
112 #define VTCON_FLAG_DETACHED	0x01
113 #define VTCON_FLAG_SIZE		0x02
114 #define VTCON_FLAG_MULTIPORT	0x04
115 
116 	/*
117 	 * Ports can be added and removed during runtime, but we have
118 	 * to allocate all the virtqueues during attach. This array is
119 	 * indexed by the port ID.
120 	 */
121 	struct vtcon_softc_port	*vtcon_ports;
122 
123 	struct task		 vtcon_ctrl_task;
124 	struct virtqueue	*vtcon_ctrl_rxvq;
125 	struct virtqueue	*vtcon_ctrl_txvq;
126 	struct mtx		 vtcon_ctrl_tx_mtx;
127 };
128 
129 #define VTCON_LOCK(_sc)			mtx_lock(&(_sc)->vtcon_mtx)
130 #define VTCON_UNLOCK(_sc)		mtx_unlock(&(_sc)->vtcon_mtx)
131 #define VTCON_LOCK_ASSERT(_sc)		\
132     mtx_assert(&(_sc)->vtcon_mtx, MA_OWNED)
133 #define VTCON_LOCK_ASSERT_NOTOWNED(_sc)	\
134     mtx_assert(&(_sc)->vtcon_mtx, MA_NOTOWNED)
135 
136 #define VTCON_CTRL_TX_LOCK(_sc)		mtx_lock(&(_sc)->vtcon_ctrl_tx_mtx)
137 #define VTCON_CTRL_TX_UNLOCK(_sc)	mtx_unlock(&(_sc)->vtcon_ctrl_tx_mtx)
138 
139 #define VTCON_ASSERT_VALID_PORTID(_sc, _id)			\
140     KASSERT((_id) >= 0 && (_id) < (_sc)->vtcon_max_ports,	\
141         ("%s: port ID %d out of range", __func__, _id))
142 
143 #define VTCON_FEATURES  VIRTIO_CONSOLE_F_MULTIPORT
144 
145 static struct virtio_feature_desc vtcon_feature_desc[] = {
146 	{ VIRTIO_CONSOLE_F_SIZE,	"ConsoleSize"	},
147 	{ VIRTIO_CONSOLE_F_MULTIPORT,	"MultiplePorts"	},
148 	{ VIRTIO_CONSOLE_F_EMERG_WRITE,	"EmergencyWrite" },
149 	{ 0, NULL }
150 };
151 
152 static int	 vtcon_modevent(module_t, int, void *);
153 static void	 vtcon_drain_all(void);
154 
155 static int	 vtcon_probe(device_t);
156 static int	 vtcon_attach(device_t);
157 static int	 vtcon_detach(device_t);
158 static int	 vtcon_config_change(device_t);
159 
160 static int	 vtcon_setup_features(struct vtcon_softc *);
161 static int	 vtcon_negotiate_features(struct vtcon_softc *);
162 static int	 vtcon_alloc_scports(struct vtcon_softc *);
163 static int	 vtcon_alloc_virtqueues(struct vtcon_softc *);
164 static void	 vtcon_read_config(struct vtcon_softc *,
165 		     struct virtio_console_config *);
166 
167 static void	 vtcon_determine_max_ports(struct vtcon_softc *,
168 		     struct virtio_console_config *);
169 static void	 vtcon_destroy_ports(struct vtcon_softc *);
170 static void	 vtcon_stop(struct vtcon_softc *);
171 
172 static int	 vtcon_ctrl_event_enqueue(struct vtcon_softc *,
173 		     struct virtio_console_control *);
174 static int	 vtcon_ctrl_event_create(struct vtcon_softc *);
175 static void	 vtcon_ctrl_event_requeue(struct vtcon_softc *,
176 		     struct virtio_console_control *);
177 static int	 vtcon_ctrl_event_populate(struct vtcon_softc *);
178 static void	 vtcon_ctrl_event_drain(struct vtcon_softc *);
179 static int	 vtcon_ctrl_init(struct vtcon_softc *);
180 static void	 vtcon_ctrl_deinit(struct vtcon_softc *);
181 static void	 vtcon_ctrl_port_add_event(struct vtcon_softc *, int);
182 static void	 vtcon_ctrl_port_remove_event(struct vtcon_softc *, int);
183 static void	 vtcon_ctrl_port_console_event(struct vtcon_softc *, int);
184 static void	 vtcon_ctrl_port_open_event(struct vtcon_softc *, int);
185 static void	 vtcon_ctrl_port_name_event(struct vtcon_softc *, int,
186 		     const char *, size_t);
187 static void	 vtcon_ctrl_process_event(struct vtcon_softc *,
188 		     struct virtio_console_control *, void *, size_t);
189 static void	 vtcon_ctrl_task_cb(void *, int);
190 static void	 vtcon_ctrl_event_intr(void *);
191 static void	 vtcon_ctrl_poll(struct vtcon_softc *,
192 		     struct virtio_console_control *control);
193 static void	 vtcon_ctrl_send_control(struct vtcon_softc *, uint32_t,
194 		     uint16_t, uint16_t);
195 
196 static int	 vtcon_port_enqueue_buf(struct vtcon_port *, void *, size_t);
197 static int	 vtcon_port_create_buf(struct vtcon_port *);
198 static void	 vtcon_port_requeue_buf(struct vtcon_port *, void *);
199 static int	 vtcon_port_populate(struct vtcon_port *);
200 static void	 vtcon_port_destroy(struct vtcon_port *);
201 static int	 vtcon_port_create(struct vtcon_softc *, int);
202 static void	 vtcon_port_dev_alias(struct vtcon_port *, const char *,
203 		     size_t);
204 static void	 vtcon_port_drain_bufs(struct virtqueue *);
205 static void	 vtcon_port_drain(struct vtcon_port *);
206 static void	 vtcon_port_teardown(struct vtcon_port *);
207 static void	 vtcon_port_change_size(struct vtcon_port *, uint16_t,
208 		     uint16_t);
209 static void	 vtcon_port_update_console_size(struct vtcon_softc *);
210 static void	 vtcon_port_enable_intr(struct vtcon_port *);
211 static void	 vtcon_port_disable_intr(struct vtcon_port *);
212 static void	 vtcon_port_in(struct vtcon_port *);
213 static void	 vtcon_port_intr(void *);
214 static void	 vtcon_port_out(struct vtcon_port *, void *, int);
215 static void	 vtcon_port_submit_event(struct vtcon_port *, uint16_t,
216 		     uint16_t);
217 
218 static int	 vtcon_tty_open(struct tty *);
219 static void	 vtcon_tty_close(struct tty *);
220 static void	 vtcon_tty_outwakeup(struct tty *);
221 static void	 vtcon_tty_free(void *);
222 
223 static void	 vtcon_get_console_size(struct vtcon_softc *, uint16_t *,
224 		     uint16_t *);
225 
226 static void	 vtcon_enable_interrupts(struct vtcon_softc *);
227 static void	 vtcon_disable_interrupts(struct vtcon_softc *);
228 
229 #define vtcon_modern(_sc) (((_sc)->vtcon_features & VIRTIO_F_VERSION_1) != 0)
230 #define vtcon_htog16(_sc, _val)	virtio_htog16(vtcon_modern(_sc), _val)
231 #define vtcon_htog32(_sc, _val)	virtio_htog32(vtcon_modern(_sc), _val)
232 #define vtcon_htog64(_sc, _val)	virtio_htog64(vtcon_modern(_sc), _val)
233 #define vtcon_gtoh16(_sc, _val)	virtio_gtoh16(vtcon_modern(_sc), _val)
234 #define vtcon_gtoh32(_sc, _val)	virtio_gtoh32(vtcon_modern(_sc), _val)
235 #define vtcon_gtoh64(_sc, _val)	virtio_gtoh64(vtcon_modern(_sc), _val)
236 
237 static int	 vtcon_pending_free;
238 
239 static struct ttydevsw vtcon_tty_class = {
240 	.tsw_flags	= 0,
241 	.tsw_open	= vtcon_tty_open,
242 	.tsw_close	= vtcon_tty_close,
243 	.tsw_outwakeup	= vtcon_tty_outwakeup,
244 	.tsw_free	= vtcon_tty_free,
245 };
246 
247 static device_method_t vtcon_methods[] = {
248 	/* Device methods. */
249 	DEVMETHOD(device_probe,		vtcon_probe),
250 	DEVMETHOD(device_attach,	vtcon_attach),
251 	DEVMETHOD(device_detach,	vtcon_detach),
252 
253 	/* VirtIO methods. */
254 	DEVMETHOD(virtio_config_change,	vtcon_config_change),
255 
256 	DEVMETHOD_END
257 };
258 
259 static driver_t vtcon_driver = {
260 	"vtcon",
261 	vtcon_methods,
262 	sizeof(struct vtcon_softc)
263 };
264 
265 VIRTIO_DRIVER_MODULE(virtio_console, vtcon_driver, vtcon_modevent, NULL);
266 MODULE_VERSION(virtio_console, 1);
267 MODULE_DEPEND(virtio_console, virtio, 1, 1, 1);
268 
269 VIRTIO_SIMPLE_PNPINFO(virtio_console, VIRTIO_ID_CONSOLE,
270     "VirtIO Console Adapter");
271 
272 static int
273 vtcon_modevent(module_t mod, int type, void *unused)
274 {
275 	int error;
276 
277 	switch (type) {
278 	case MOD_LOAD:
279 		error = 0;
280 		break;
281 	case MOD_QUIESCE:
282 		error = 0;
283 		break;
284 	case MOD_UNLOAD:
285 		vtcon_drain_all();
286 		error = 0;
287 		break;
288 	case MOD_SHUTDOWN:
289 		error = 0;
290 		break;
291 	default:
292 		error = EOPNOTSUPP;
293 		break;
294 	}
295 
296 	return (error);
297 }
298 
299 static void
300 vtcon_drain_all(void)
301 {
302 	int first;
303 
304 	for (first = 1; vtcon_pending_free != 0; first = 0) {
305 		if (first != 0) {
306 			printf("virtio_console: Waiting for all detached TTY "
307 			    "devices to have open fds closed.\n");
308 		}
309 		pause("vtcondra", hz);
310 	}
311 }
312 
313 static int
314 vtcon_probe(device_t dev)
315 {
316 	return (VIRTIO_SIMPLE_PROBE(dev, virtio_console));
317 }
318 
319 static int
320 vtcon_attach(device_t dev)
321 {
322 	struct vtcon_softc *sc;
323 	struct virtio_console_config concfg;
324 	int error;
325 
326 	sc = device_get_softc(dev);
327 	sc->vtcon_dev = dev;
328 	virtio_set_feature_desc(dev, vtcon_feature_desc);
329 
330 	mtx_init(&sc->vtcon_mtx, "vtconmtx", NULL, MTX_DEF);
331 	mtx_init(&sc->vtcon_ctrl_tx_mtx, "vtconctrlmtx", NULL, MTX_DEF);
332 
333 	error = vtcon_setup_features(sc);
334 	if (error) {
335 		device_printf(dev, "cannot setup features\n");
336 		goto fail;
337 	}
338 
339 	vtcon_read_config(sc, &concfg);
340 	vtcon_determine_max_ports(sc, &concfg);
341 
342 	error = vtcon_alloc_scports(sc);
343 	if (error) {
344 		device_printf(dev, "cannot allocate softc port structures\n");
345 		goto fail;
346 	}
347 
348 	error = vtcon_alloc_virtqueues(sc);
349 	if (error) {
350 		device_printf(dev, "cannot allocate virtqueues\n");
351 		goto fail;
352 	}
353 
354 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
355 		TASK_INIT(&sc->vtcon_ctrl_task, 0, vtcon_ctrl_task_cb, sc);
356 		error = vtcon_ctrl_init(sc);
357 		if (error)
358 			goto fail;
359 	} else {
360 		error = vtcon_port_create(sc, 0);
361 		if (error)
362 			goto fail;
363 		if (sc->vtcon_flags & VTCON_FLAG_SIZE)
364 			vtcon_port_update_console_size(sc);
365 	}
366 
367 	error = virtio_setup_intr(dev, INTR_TYPE_TTY);
368 	if (error) {
369 		device_printf(dev, "cannot setup virtqueue interrupts\n");
370 		goto fail;
371 	}
372 
373 	vtcon_enable_interrupts(sc);
374 
375 	vtcon_ctrl_send_control(sc, VIRTIO_CONSOLE_BAD_ID,
376 	    VIRTIO_CONSOLE_DEVICE_READY, 1);
377 
378 fail:
379 	if (error)
380 		vtcon_detach(dev);
381 
382 	return (error);
383 }
384 
385 static int
386 vtcon_detach(device_t dev)
387 {
388 	struct vtcon_softc *sc;
389 
390 	sc = device_get_softc(dev);
391 
392 	VTCON_LOCK(sc);
393 	sc->vtcon_flags |= VTCON_FLAG_DETACHED;
394 	if (device_is_attached(dev))
395 		vtcon_stop(sc);
396 	VTCON_UNLOCK(sc);
397 
398 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
399 		taskqueue_drain(taskqueue_thread, &sc->vtcon_ctrl_task);
400 		vtcon_ctrl_deinit(sc);
401 	}
402 
403 	vtcon_destroy_ports(sc);
404 	mtx_destroy(&sc->vtcon_mtx);
405 	mtx_destroy(&sc->vtcon_ctrl_tx_mtx);
406 
407 	return (0);
408 }
409 
410 static int
411 vtcon_config_change(device_t dev)
412 {
413 	struct vtcon_softc *sc;
414 
415 	sc = device_get_softc(dev);
416 
417 	/*
418 	 * When the multiport feature is negotiated, all configuration
419 	 * changes are done through control virtqueue events.
420 	 */
421 	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0) {
422 		if (sc->vtcon_flags & VTCON_FLAG_SIZE)
423 			vtcon_port_update_console_size(sc);
424 	}
425 
426 	return (0);
427 }
428 
429 static int
430 vtcon_negotiate_features(struct vtcon_softc *sc)
431 {
432 	device_t dev;
433 	uint64_t features;
434 
435 	dev = sc->vtcon_dev;
436 	features = VTCON_FEATURES;
437 
438 	sc->vtcon_features = virtio_negotiate_features(dev, features);
439 	return (virtio_finalize_features(dev));
440 }
441 
442 static int
443 vtcon_setup_features(struct vtcon_softc *sc)
444 {
445 	device_t dev;
446 	int error;
447 
448 	dev = sc->vtcon_dev;
449 
450 	error = vtcon_negotiate_features(sc);
451 	if (error)
452 		return (error);
453 
454 	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_SIZE))
455 		sc->vtcon_flags |= VTCON_FLAG_SIZE;
456 	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_MULTIPORT))
457 		sc->vtcon_flags |= VTCON_FLAG_MULTIPORT;
458 
459 	return (0);
460 }
461 
462 #define VTCON_GET_CONFIG(_dev, _feature, _field, _cfg)			\
463 	if (virtio_with_feature(_dev, _feature)) {			\
464 		virtio_read_device_config(_dev,				\
465 		    offsetof(struct virtio_console_config, _field),	\
466 		    &(_cfg)->_field, sizeof((_cfg)->_field));		\
467 	}
468 
469 static void
470 vtcon_read_config(struct vtcon_softc *sc, struct virtio_console_config *concfg)
471 {
472 	device_t dev;
473 
474 	dev = sc->vtcon_dev;
475 
476 	bzero(concfg, sizeof(struct virtio_console_config));
477 
478 	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, cols, concfg);
479 	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_SIZE, rows, concfg);
480 	VTCON_GET_CONFIG(dev, VIRTIO_CONSOLE_F_MULTIPORT, max_nr_ports, concfg);
481 }
482 
483 #undef VTCON_GET_CONFIG
484 
485 static int
486 vtcon_alloc_scports(struct vtcon_softc *sc)
487 {
488 	struct vtcon_softc_port *scport;
489 	int max, i;
490 
491 	max = sc->vtcon_max_ports;
492 
493 	sc->vtcon_ports = malloc(sizeof(struct vtcon_softc_port) * max,
494 	    M_DEVBUF, M_NOWAIT | M_ZERO);
495 	if (sc->vtcon_ports == NULL)
496 		return (ENOMEM);
497 
498 	for (i = 0; i < max; i++) {
499 		scport = &sc->vtcon_ports[i];
500 		scport->vcsp_sc = sc;
501 	}
502 
503 	return (0);
504 }
505 
506 static int
507 vtcon_alloc_virtqueues(struct vtcon_softc *sc)
508 {
509 	device_t dev;
510 	struct vq_alloc_info *info;
511 	struct vtcon_softc_port *scport;
512 	int i, idx, portidx, nvqs, error;
513 
514 	dev = sc->vtcon_dev;
515 
516 	nvqs = sc->vtcon_max_ports * 2;
517 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
518 		nvqs += 2;
519 
520 	info = malloc(sizeof(struct vq_alloc_info) * nvqs, M_TEMP, M_NOWAIT);
521 	if (info == NULL)
522 		return (ENOMEM);
523 
524 	for (i = 0, idx = 0, portidx = 0; i < nvqs / 2; i++, idx += 2) {
525 		if (i == 1) {
526 			/* The control virtqueues are after the first port. */
527 			VQ_ALLOC_INFO_INIT(&info[idx], 0,
528 			    vtcon_ctrl_event_intr, sc, &sc->vtcon_ctrl_rxvq,
529 			    "%s-control rx", device_get_nameunit(dev));
530 			VQ_ALLOC_INFO_INIT(&info[idx+1], 0,
531 			    NULL, sc, &sc->vtcon_ctrl_txvq,
532 			    "%s-control tx", device_get_nameunit(dev));
533 			continue;
534 		}
535 
536 		scport = &sc->vtcon_ports[portidx];
537 
538 		VQ_ALLOC_INFO_INIT(&info[idx], 0, vtcon_port_intr,
539 		    scport, &scport->vcsp_invq, "%s-port%d in",
540 		    device_get_nameunit(dev), i);
541 		VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL,
542 		    NULL, &scport->vcsp_outvq, "%s-port%d out",
543 		    device_get_nameunit(dev), i);
544 
545 		portidx++;
546 	}
547 
548 	error = virtio_alloc_virtqueues(dev, 0, nvqs, info);
549 	free(info, M_TEMP);
550 
551 	return (error);
552 }
553 
554 static void
555 vtcon_determine_max_ports(struct vtcon_softc *sc,
556     struct virtio_console_config *concfg)
557 {
558 
559 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT) {
560 		sc->vtcon_max_ports =
561 		    min(concfg->max_nr_ports, VTCON_MAX_PORTS);
562 		if (sc->vtcon_max_ports == 0)
563 			sc->vtcon_max_ports = 1;
564 	} else
565 		sc->vtcon_max_ports = 1;
566 }
567 
568 static void
569 vtcon_destroy_ports(struct vtcon_softc *sc)
570 {
571 	struct vtcon_softc_port *scport;
572 	struct vtcon_port *port;
573 	struct virtqueue *vq;
574 	int i;
575 
576 	if (sc->vtcon_ports == NULL)
577 		return;
578 
579 	VTCON_LOCK(sc);
580 	for (i = 0; i < sc->vtcon_max_ports; i++) {
581 		scport = &sc->vtcon_ports[i];
582 
583 		port = scport->vcsp_port;
584 		if (port != NULL) {
585 			scport->vcsp_port = NULL;
586 			VTCON_PORT_LOCK(port);
587 			VTCON_UNLOCK(sc);
588 			vtcon_port_teardown(port);
589 			VTCON_LOCK(sc);
590 		}
591 
592 		vq = scport->vcsp_invq;
593 		if (vq != NULL)
594 			vtcon_port_drain_bufs(vq);
595 	}
596 	VTCON_UNLOCK(sc);
597 
598 	free(sc->vtcon_ports, M_DEVBUF);
599 	sc->vtcon_ports = NULL;
600 }
601 
602 static void
603 vtcon_stop(struct vtcon_softc *sc)
604 {
605 
606 	vtcon_disable_interrupts(sc);
607 	virtio_stop(sc->vtcon_dev);
608 }
609 
610 static int
611 vtcon_ctrl_event_enqueue(struct vtcon_softc *sc,
612     struct virtio_console_control *control)
613 {
614 	struct sglist_seg segs[2];
615 	struct sglist sg;
616 	struct virtqueue *vq;
617 	int error __diagused;
618 
619 	vq = sc->vtcon_ctrl_rxvq;
620 
621 	sglist_init(&sg, 2, segs);
622 	error = sglist_append(&sg, control, VTCON_CTRL_BUFSZ);
623 	KASSERT(error == 0, ("%s: error %d adding control to sglist",
624 	    __func__, error));
625 
626 	return (virtqueue_enqueue(vq, control, &sg, 0, sg.sg_nseg));
627 }
628 
629 static int
630 vtcon_ctrl_event_create(struct vtcon_softc *sc)
631 {
632 	struct virtio_console_control *control;
633 	int error;
634 
635 	control = malloc(VTCON_CTRL_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT);
636 	if (control == NULL)
637 		return (ENOMEM);
638 
639 	error = vtcon_ctrl_event_enqueue(sc, control);
640 	if (error)
641 		free(control, M_DEVBUF);
642 
643 	return (error);
644 }
645 
646 static void
647 vtcon_ctrl_event_requeue(struct vtcon_softc *sc,
648     struct virtio_console_control *control)
649 {
650 	int error __diagused;
651 
652 	bzero(control, VTCON_CTRL_BUFSZ);
653 
654 	error = vtcon_ctrl_event_enqueue(sc, control);
655 	KASSERT(error == 0,
656 	    ("%s: cannot requeue control buffer %d", __func__, error));
657 }
658 
659 static int
660 vtcon_ctrl_event_populate(struct vtcon_softc *sc)
661 {
662 	struct virtqueue *vq;
663 	int nbufs, error;
664 
665 	vq = sc->vtcon_ctrl_rxvq;
666 	error = ENOSPC;
667 
668 	for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
669 		error = vtcon_ctrl_event_create(sc);
670 		if (error)
671 			break;
672 	}
673 
674 	if (nbufs > 0) {
675 		virtqueue_notify(vq);
676 		error = 0;
677 	}
678 
679 	return (error);
680 }
681 
682 static void
683 vtcon_ctrl_event_drain(struct vtcon_softc *sc)
684 {
685 	struct virtio_console_control *control;
686 	struct virtqueue *vq;
687 	int last;
688 
689 	vq = sc->vtcon_ctrl_rxvq;
690 	last = 0;
691 
692 	if (vq == NULL)
693 		return;
694 
695 	VTCON_LOCK(sc);
696 	while ((control = virtqueue_drain(vq, &last)) != NULL)
697 		free(control, M_DEVBUF);
698 	VTCON_UNLOCK(sc);
699 }
700 
701 static int
702 vtcon_ctrl_init(struct vtcon_softc *sc)
703 {
704 	int error;
705 
706 	error = vtcon_ctrl_event_populate(sc);
707 
708 	return (error);
709 }
710 
711 static void
712 vtcon_ctrl_deinit(struct vtcon_softc *sc)
713 {
714 
715 	vtcon_ctrl_event_drain(sc);
716 }
717 
718 static void
719 vtcon_ctrl_port_add_event(struct vtcon_softc *sc, int id)
720 {
721 	device_t dev;
722 	int error;
723 
724 	dev = sc->vtcon_dev;
725 
726 	/* This single thread only way for ports to be created. */
727 	if (sc->vtcon_ports[id].vcsp_port != NULL) {
728 		device_printf(dev, "%s: adding port %d, but already exists\n",
729 		    __func__, id);
730 		return;
731 	}
732 
733 	error = vtcon_port_create(sc, id);
734 	if (error) {
735 		device_printf(dev, "%s: cannot create port %d: %d\n",
736 		    __func__, id, error);
737 		vtcon_ctrl_send_control(sc, id, VIRTIO_CONSOLE_PORT_READY, 0);
738 		return;
739 	}
740 }
741 
742 static void
743 vtcon_ctrl_port_remove_event(struct vtcon_softc *sc, int id)
744 {
745 	device_t dev;
746 	struct vtcon_softc_port *scport;
747 	struct vtcon_port *port;
748 
749 	dev = sc->vtcon_dev;
750 	scport = &sc->vtcon_ports[id];
751 
752 	VTCON_LOCK(sc);
753 	port = scport->vcsp_port;
754 	if (port == NULL) {
755 		VTCON_UNLOCK(sc);
756 		device_printf(dev, "%s: remove port %d, but does not exist\n",
757 		    __func__, id);
758 		return;
759 	}
760 
761 	scport->vcsp_port = NULL;
762 	VTCON_PORT_LOCK(port);
763 	VTCON_UNLOCK(sc);
764 	vtcon_port_teardown(port);
765 }
766 
767 static void
768 vtcon_ctrl_port_console_event(struct vtcon_softc *sc, int id)
769 {
770 	device_t dev;
771 	struct vtcon_softc_port *scport;
772 	struct vtcon_port *port;
773 
774 	dev = sc->vtcon_dev;
775 	scport = &sc->vtcon_ports[id];
776 
777 	VTCON_LOCK(sc);
778 	port = scport->vcsp_port;
779 	if (port == NULL) {
780 		VTCON_UNLOCK(sc);
781 		device_printf(dev, "%s: console port %d, but does not exist\n",
782 		    __func__, id);
783 		return;
784 	}
785 
786 	VTCON_PORT_LOCK(port);
787 	VTCON_UNLOCK(sc);
788 	port->vtcport_flags |= VTCON_PORT_FLAG_CONSOLE;
789 	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
790 	VTCON_PORT_UNLOCK(port);
791 }
792 
793 static void
794 vtcon_ctrl_port_open_event(struct vtcon_softc *sc, int id)
795 {
796 	device_t dev;
797 	struct vtcon_softc_port *scport;
798 	struct vtcon_port *port;
799 
800 	dev = sc->vtcon_dev;
801 	scport = &sc->vtcon_ports[id];
802 
803 	VTCON_LOCK(sc);
804 	port = scport->vcsp_port;
805 	if (port == NULL) {
806 		VTCON_UNLOCK(sc);
807 		device_printf(dev, "%s: open port %d, but does not exist\n",
808 		    __func__, id);
809 		return;
810 	}
811 
812 	VTCON_PORT_LOCK(port);
813 	VTCON_UNLOCK(sc);
814 	vtcon_port_enable_intr(port);
815 	VTCON_PORT_UNLOCK(port);
816 }
817 
818 static void
819 vtcon_ctrl_port_name_event(struct vtcon_softc *sc, int id, const char *name,
820     size_t len)
821 {
822 	device_t dev;
823 	struct vtcon_softc_port *scport;
824 	struct vtcon_port *port;
825 
826 	dev = sc->vtcon_dev;
827 	scport = &sc->vtcon_ports[id];
828 
829 	/*
830 	 * The VirtIO specification says the NUL terminator is not included in
831 	 * the length, but QEMU includes it. Adjust the length if needed.
832 	 */
833 	if (name == NULL || len == 0)
834 		return;
835 	if (name[len - 1] == '\0') {
836 		len--;
837 		if (len == 0)
838 			return;
839 	}
840 
841 	VTCON_LOCK(sc);
842 	port = scport->vcsp_port;
843 	if (port == NULL) {
844 		VTCON_UNLOCK(sc);
845 		device_printf(dev, "%s: name port %d, but does not exist\n",
846 		    __func__, id);
847 		return;
848 	}
849 
850 	VTCON_PORT_LOCK(port);
851 	VTCON_UNLOCK(sc);
852 	vtcon_port_dev_alias(port, name, len);
853 	VTCON_PORT_UNLOCK(port);
854 }
855 
856 static void
857 vtcon_ctrl_process_event(struct vtcon_softc *sc,
858     struct virtio_console_control *control, void *data, size_t data_len)
859 {
860 	device_t dev;
861 	uint32_t id;
862 	uint16_t event;
863 
864 	dev = sc->vtcon_dev;
865 	id = vtcon_htog32(sc, control->id);
866 	event = vtcon_htog16(sc, control->event);
867 
868 	if (id >= sc->vtcon_max_ports) {
869 		device_printf(dev, "%s: event %d invalid port ID %d\n",
870 		    __func__, event, id);
871 		return;
872 	}
873 
874 	switch (event) {
875 	case VIRTIO_CONSOLE_PORT_ADD:
876 		vtcon_ctrl_port_add_event(sc, id);
877 		break;
878 
879 	case VIRTIO_CONSOLE_PORT_REMOVE:
880 		vtcon_ctrl_port_remove_event(sc, id);
881 		break;
882 
883 	case VIRTIO_CONSOLE_CONSOLE_PORT:
884 		vtcon_ctrl_port_console_event(sc, id);
885 		break;
886 
887 	case VIRTIO_CONSOLE_RESIZE:
888 		break;
889 
890 	case VIRTIO_CONSOLE_PORT_OPEN:
891 		vtcon_ctrl_port_open_event(sc, id);
892 		break;
893 
894 	case VIRTIO_CONSOLE_PORT_NAME:
895 		vtcon_ctrl_port_name_event(sc, id, (const char *)data, data_len);
896 		break;
897 	}
898 }
899 
900 static void
901 vtcon_ctrl_task_cb(void *xsc, int pending)
902 {
903 	struct vtcon_softc *sc;
904 	struct virtqueue *vq;
905 	struct virtio_console_control *control;
906 	void *data;
907 	size_t data_len;
908 	int detached;
909 	uint32_t len;
910 
911 	sc = xsc;
912 	vq = sc->vtcon_ctrl_rxvq;
913 
914 	VTCON_LOCK(sc);
915 
916 	while ((detached = (sc->vtcon_flags & VTCON_FLAG_DETACHED)) == 0) {
917 		control = virtqueue_dequeue(vq, &len);
918 		if (control == NULL)
919 			break;
920 
921 		if (len > sizeof(struct virtio_console_control)) {
922 			data = (void *) &control[1];
923 			data_len = len - sizeof(struct virtio_console_control);
924 		} else {
925 			data = NULL;
926 			data_len = 0;
927 		}
928 
929 		VTCON_UNLOCK(sc);
930 		vtcon_ctrl_process_event(sc, control, data, data_len);
931 		VTCON_LOCK(sc);
932 		vtcon_ctrl_event_requeue(sc, control);
933 	}
934 
935 	if (!detached) {
936 		virtqueue_notify(vq);
937 		if (virtqueue_enable_intr(vq) != 0)
938 			taskqueue_enqueue(taskqueue_thread,
939 			    &sc->vtcon_ctrl_task);
940 	}
941 
942 	VTCON_UNLOCK(sc);
943 }
944 
945 static void
946 vtcon_ctrl_event_intr(void *xsc)
947 {
948 	struct vtcon_softc *sc;
949 
950 	sc = xsc;
951 
952 	/*
953 	 * Only some events require us to potentially block, but it
954 	 * easier to just defer all event handling to the taskqueue.
955 	 */
956 	taskqueue_enqueue(taskqueue_thread, &sc->vtcon_ctrl_task);
957 }
958 
959 static void
960 vtcon_ctrl_poll(struct vtcon_softc *sc,
961     struct virtio_console_control *control)
962 {
963 	struct sglist_seg segs[2];
964 	struct sglist sg;
965 	struct virtqueue *vq;
966 	int error;
967 
968 	vq = sc->vtcon_ctrl_txvq;
969 
970 	sglist_init(&sg, 2, segs);
971 	error = sglist_append(&sg, control,
972 	    sizeof(struct virtio_console_control));
973 	KASSERT(error == 0, ("%s: error %d adding control to sglist",
974 	    __func__, error));
975 
976 	/*
977 	 * We cannot use the softc lock to serialize access to this
978 	 * virtqueue since this is called from the tty layer with the
979 	 * port lock held. Acquiring the softc would violate our lock
980 	 * ordering.
981 	 */
982 	VTCON_CTRL_TX_LOCK(sc);
983 	KASSERT(virtqueue_empty(vq),
984 	    ("%s: virtqueue is not emtpy", __func__));
985 	error = virtqueue_enqueue(vq, control, &sg, sg.sg_nseg, 0);
986 	if (error == 0) {
987 		virtqueue_notify(vq);
988 		virtqueue_poll(vq, NULL);
989 	}
990 	VTCON_CTRL_TX_UNLOCK(sc);
991 }
992 
993 static void
994 vtcon_ctrl_send_control(struct vtcon_softc *sc, uint32_t portid,
995     uint16_t event, uint16_t value)
996 {
997 	struct virtio_console_control control;
998 
999 	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0)
1000 		return;
1001 
1002 	control.id = vtcon_gtoh32(sc, portid);
1003 	control.event = vtcon_gtoh16(sc, event);
1004 	control.value = vtcon_gtoh16(sc, value);
1005 
1006 	vtcon_ctrl_poll(sc, &control);
1007 }
1008 
1009 static int
1010 vtcon_port_enqueue_buf(struct vtcon_port *port, void *buf, size_t len)
1011 {
1012 	struct sglist_seg segs[2];
1013 	struct sglist sg;
1014 	struct virtqueue *vq;
1015 	int error;
1016 
1017 	vq = port->vtcport_invq;
1018 
1019 	sglist_init(&sg, 2, segs);
1020 	error = sglist_append(&sg, buf, len);
1021 	KASSERT(error == 0,
1022 	    ("%s: error %d adding buffer to sglist", __func__, error));
1023 
1024 	error = virtqueue_enqueue(vq, buf, &sg, 0, sg.sg_nseg);
1025 
1026 	return (error);
1027 }
1028 
1029 static int
1030 vtcon_port_create_buf(struct vtcon_port *port)
1031 {
1032 	void *buf;
1033 	int error;
1034 
1035 	buf = malloc(VTCON_BULK_BUFSZ, M_DEVBUF, M_ZERO | M_NOWAIT);
1036 	if (buf == NULL)
1037 		return (ENOMEM);
1038 
1039 	error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ);
1040 	if (error)
1041 		free(buf, M_DEVBUF);
1042 
1043 	return (error);
1044 }
1045 
1046 static void
1047 vtcon_port_requeue_buf(struct vtcon_port *port, void *buf)
1048 {
1049 	int error __diagused;
1050 
1051 	error = vtcon_port_enqueue_buf(port, buf, VTCON_BULK_BUFSZ);
1052 	KASSERT(error == 0,
1053 	    ("%s: cannot requeue input buffer %d", __func__, error));
1054 }
1055 
1056 static int
1057 vtcon_port_populate(struct vtcon_port *port)
1058 {
1059 	struct virtqueue *vq;
1060 	int nbufs, error;
1061 
1062 	vq = port->vtcport_invq;
1063 	error = ENOSPC;
1064 
1065 	for (nbufs = 0; !virtqueue_full(vq); nbufs++) {
1066 		error = vtcon_port_create_buf(port);
1067 		if (error)
1068 			break;
1069 	}
1070 
1071 	if (nbufs > 0) {
1072 		virtqueue_notify(vq);
1073 		error = 0;
1074 	}
1075 
1076 	return (error);
1077 }
1078 
1079 static void
1080 vtcon_port_destroy(struct vtcon_port *port)
1081 {
1082 
1083 	port->vtcport_sc = NULL;
1084 	port->vtcport_scport = NULL;
1085 	port->vtcport_invq = NULL;
1086 	port->vtcport_outvq = NULL;
1087 	port->vtcport_id = -1;
1088 	mtx_destroy(&port->vtcport_mtx);
1089 	free(port, M_DEVBUF);
1090 }
1091 
1092 static int
1093 vtcon_port_init_vqs(struct vtcon_port *port)
1094 {
1095 	struct vtcon_softc_port *scport;
1096 	int error;
1097 
1098 	scport = port->vtcport_scport;
1099 
1100 	port->vtcport_invq = scport->vcsp_invq;
1101 	port->vtcport_outvq = scport->vcsp_outvq;
1102 
1103 	/*
1104 	 * Free any data left over from when this virtqueue was in use by a
1105 	 * prior port. We have not yet notified the host that the port is
1106 	 * ready, so assume nothing in the virtqueue can be for us.
1107 	 */
1108 	vtcon_port_drain(port);
1109 
1110 	KASSERT(virtqueue_empty(port->vtcport_invq),
1111 	    ("%s: in virtqueue is not empty", __func__));
1112 	KASSERT(virtqueue_empty(port->vtcport_outvq),
1113 	    ("%s: out virtqueue is not empty", __func__));
1114 
1115 	error = vtcon_port_populate(port);
1116 	if (error)
1117 		return (error);
1118 
1119 	return (0);
1120 }
1121 
1122 static int
1123 vtcon_port_create(struct vtcon_softc *sc, int id)
1124 {
1125 	device_t dev;
1126 	struct vtcon_softc_port *scport;
1127 	struct vtcon_port *port;
1128 	int error;
1129 
1130 	dev = sc->vtcon_dev;
1131 	scport = &sc->vtcon_ports[id];
1132 
1133 	VTCON_ASSERT_VALID_PORTID(sc, id);
1134 	MPASS(scport->vcsp_port == NULL);
1135 
1136 	port = malloc(sizeof(struct vtcon_port), M_DEVBUF, M_NOWAIT | M_ZERO);
1137 	if (port == NULL)
1138 		return (ENOMEM);
1139 
1140 	port->vtcport_sc = sc;
1141 	port->vtcport_scport = scport;
1142 	port->vtcport_id = id;
1143 	mtx_init(&port->vtcport_mtx, "vtcpmtx", NULL, MTX_DEF);
1144 	port->vtcport_tty = tty_alloc_mutex(&vtcon_tty_class, port,
1145 	    &port->vtcport_mtx);
1146 
1147 	error = vtcon_port_init_vqs(port);
1148 	if (error) {
1149 		VTCON_PORT_LOCK(port);
1150 		vtcon_port_teardown(port);
1151 		return (error);
1152 	}
1153 
1154 	VTCON_LOCK(sc);
1155 	VTCON_PORT_LOCK(port);
1156 	scport->vcsp_port = port;
1157 	vtcon_port_enable_intr(port);
1158 	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_READY, 1);
1159 	VTCON_PORT_UNLOCK(port);
1160 	VTCON_UNLOCK(sc);
1161 
1162 	tty_makedev(port->vtcport_tty, NULL, "%s%r.%r", VTCON_TTY_PREFIX,
1163 	    device_get_unit(dev), id);
1164 
1165 	return (0);
1166 }
1167 
1168 static void
1169 vtcon_port_dev_alias(struct vtcon_port *port, const char *name, size_t len)
1170 {
1171 	struct vtcon_softc *sc;
1172 	struct cdev *pdev;
1173 	struct tty *tp;
1174 	int i, error;
1175 
1176 	sc = port->vtcport_sc;
1177 	tp = port->vtcport_tty;
1178 
1179 	if (port->vtcport_flags & VTCON_PORT_FLAG_ALIAS)
1180 		return;
1181 
1182 	/* Port name is UTF-8, but we can only handle ASCII. */
1183 	for (i = 0; i < len; i++) {
1184 		if (!isascii(name[i]))
1185 			return;
1186 	}
1187 
1188 	/*
1189 	 * Port name may not conform to the devfs requirements so we cannot use
1190 	 * tty_makealias() because the MAKEDEV_CHECKNAME flag must be specified.
1191 	 */
1192 	error = make_dev_alias_p(MAKEDEV_NOWAIT | MAKEDEV_CHECKNAME, &pdev,
1193 	    tp->t_dev, "%s/%*s", VTCON_TTY_ALIAS_PREFIX, (int)len, name);
1194 	if (error) {
1195 		device_printf(sc->vtcon_dev,
1196 		    "%s: cannot make dev alias (%s/%*s) error %d\n", __func__,
1197 		    VTCON_TTY_ALIAS_PREFIX, (int)len, name, error);
1198 	} else
1199 		port->vtcport_flags |= VTCON_PORT_FLAG_ALIAS;
1200 }
1201 
1202 static void
1203 vtcon_port_drain_bufs(struct virtqueue *vq)
1204 {
1205 	void *buf;
1206 	int last;
1207 
1208 	last = 0;
1209 
1210 	while ((buf = virtqueue_drain(vq, &last)) != NULL)
1211 		free(buf, M_DEVBUF);
1212 }
1213 
1214 static void
1215 vtcon_port_drain(struct vtcon_port *port)
1216 {
1217 
1218 	vtcon_port_drain_bufs(port->vtcport_invq);
1219 }
1220 
1221 static void
1222 vtcon_port_teardown(struct vtcon_port *port)
1223 {
1224 	struct tty *tp;
1225 
1226 	tp = port->vtcport_tty;
1227 
1228 	port->vtcport_flags |= VTCON_PORT_FLAG_GONE;
1229 
1230 	if (tp != NULL) {
1231 		atomic_add_int(&vtcon_pending_free, 1);
1232 		tty_rel_gone(tp);
1233 	} else
1234 		vtcon_port_destroy(port);
1235 }
1236 
1237 static void
1238 vtcon_port_change_size(struct vtcon_port *port, uint16_t cols, uint16_t rows)
1239 {
1240 	struct tty *tp;
1241 	struct winsize sz;
1242 
1243 	tp = port->vtcport_tty;
1244 
1245 	if (tp == NULL)
1246 		return;
1247 
1248 	bzero(&sz, sizeof(struct winsize));
1249 	sz.ws_col = cols;
1250 	sz.ws_row = rows;
1251 
1252 	tty_set_winsize(tp, &sz);
1253 }
1254 
1255 static void
1256 vtcon_port_update_console_size(struct vtcon_softc *sc)
1257 {
1258 	struct vtcon_port *port;
1259 	struct vtcon_softc_port *scport;
1260 	uint16_t cols, rows;
1261 
1262 	vtcon_get_console_size(sc, &cols, &rows);
1263 
1264 	/*
1265 	 * For now, assume the first (only) port is the console. Note
1266 	 * QEMU does not implement this feature yet.
1267 	 */
1268 	scport = &sc->vtcon_ports[0];
1269 
1270 	VTCON_LOCK(sc);
1271 	port = scport->vcsp_port;
1272 
1273 	if (port != NULL) {
1274 		VTCON_PORT_LOCK(port);
1275 		VTCON_UNLOCK(sc);
1276 		vtcon_port_change_size(port, cols, rows);
1277 		VTCON_PORT_UNLOCK(port);
1278 	} else
1279 		VTCON_UNLOCK(sc);
1280 }
1281 
1282 static void
1283 vtcon_port_enable_intr(struct vtcon_port *port)
1284 {
1285 
1286 	/*
1287 	 * NOTE: The out virtqueue is always polled, so its interrupt
1288 	 * kept disabled.
1289 	 */
1290 	virtqueue_enable_intr(port->vtcport_invq);
1291 }
1292 
1293 static void
1294 vtcon_port_disable_intr(struct vtcon_port *port)
1295 {
1296 
1297 	if (port->vtcport_invq != NULL)
1298 		virtqueue_disable_intr(port->vtcport_invq);
1299 	if (port->vtcport_outvq != NULL)
1300 		virtqueue_disable_intr(port->vtcport_outvq);
1301 }
1302 
1303 static void
1304 vtcon_port_in(struct vtcon_port *port)
1305 {
1306 	struct virtqueue *vq;
1307 	struct tty *tp;
1308 	char *buf;
1309 	uint32_t len;
1310 	int i, deq;
1311 
1312 	tp = port->vtcport_tty;
1313 	vq = port->vtcport_invq;
1314 
1315 again:
1316 	deq = 0;
1317 
1318 	while ((buf = virtqueue_dequeue(vq, &len)) != NULL) {
1319 		for (i = 0; i < len; i++) {
1320 #if defined(KDB)
1321 			if (port->vtcport_flags & VTCON_PORT_FLAG_CONSOLE)
1322 				kdb_alt_break(buf[i],
1323 				    &port->vtcport_alt_break_state);
1324 #endif
1325 			ttydisc_rint(tp, buf[i], 0);
1326 		}
1327 		vtcon_port_requeue_buf(port, buf);
1328 		deq++;
1329 	}
1330 	ttydisc_rint_done(tp);
1331 
1332 	if (deq > 0)
1333 		virtqueue_notify(vq);
1334 
1335 	if (virtqueue_enable_intr(vq) != 0)
1336 		goto again;
1337 }
1338 
1339 static void
1340 vtcon_port_intr(void *scportx)
1341 {
1342 	struct vtcon_softc_port *scport;
1343 	struct vtcon_softc *sc;
1344 	struct vtcon_port *port;
1345 
1346 	scport = scportx;
1347 	sc = scport->vcsp_sc;
1348 
1349 	VTCON_LOCK(sc);
1350 	port = scport->vcsp_port;
1351 	if (port == NULL) {
1352 		VTCON_UNLOCK(sc);
1353 		return;
1354 	}
1355 	VTCON_PORT_LOCK(port);
1356 	VTCON_UNLOCK(sc);
1357 	if ((port->vtcport_flags & VTCON_PORT_FLAG_GONE) == 0)
1358 		vtcon_port_in(port);
1359 	VTCON_PORT_UNLOCK(port);
1360 }
1361 
1362 static void
1363 vtcon_port_out(struct vtcon_port *port, void *buf, int bufsize)
1364 {
1365 	struct sglist_seg segs[2];
1366 	struct sglist sg;
1367 	struct virtqueue *vq;
1368 	int error;
1369 
1370 	vq = port->vtcport_outvq;
1371 	KASSERT(virtqueue_empty(vq),
1372 	    ("%s: port %p out virtqueue not emtpy", __func__, port));
1373 
1374 	sglist_init(&sg, 2, segs);
1375 	error = sglist_append(&sg, buf, bufsize);
1376 	KASSERT(error == 0, ("%s: error %d adding buffer to sglist",
1377 	    __func__, error));
1378 
1379 	error = virtqueue_enqueue(vq, buf, &sg, sg.sg_nseg, 0);
1380 	if (error == 0) {
1381 		virtqueue_notify(vq);
1382 		virtqueue_poll(vq, NULL);
1383 	}
1384 }
1385 
1386 static void
1387 vtcon_port_submit_event(struct vtcon_port *port, uint16_t event,
1388     uint16_t value)
1389 {
1390 	struct vtcon_softc *sc;
1391 
1392 	sc = port->vtcport_sc;
1393 
1394 	vtcon_ctrl_send_control(sc, port->vtcport_id, event, value);
1395 }
1396 
1397 static int
1398 vtcon_tty_open(struct tty *tp)
1399 {
1400 	struct vtcon_port *port;
1401 
1402 	port = tty_softc(tp);
1403 
1404 	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
1405 		return (ENXIO);
1406 
1407 	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 1);
1408 
1409 	return (0);
1410 }
1411 
1412 static void
1413 vtcon_tty_close(struct tty *tp)
1414 {
1415 	struct vtcon_port *port;
1416 
1417 	port = tty_softc(tp);
1418 
1419 	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
1420 		return;
1421 
1422 	vtcon_port_submit_event(port, VIRTIO_CONSOLE_PORT_OPEN, 0);
1423 }
1424 
1425 static void
1426 vtcon_tty_outwakeup(struct tty *tp)
1427 {
1428 	struct vtcon_port *port;
1429 	char buf[VTCON_BULK_BUFSZ];
1430 	int len;
1431 
1432 	port = tty_softc(tp);
1433 
1434 	if (port->vtcport_flags & VTCON_PORT_FLAG_GONE)
1435 		return;
1436 
1437 	while ((len = ttydisc_getc(tp, buf, sizeof(buf))) != 0)
1438 		vtcon_port_out(port, buf, len);
1439 }
1440 
1441 static void
1442 vtcon_tty_free(void *xport)
1443 {
1444 	struct vtcon_port *port;
1445 
1446 	port = xport;
1447 
1448 	vtcon_port_destroy(port);
1449 	atomic_subtract_int(&vtcon_pending_free, 1);
1450 }
1451 
1452 static void
1453 vtcon_get_console_size(struct vtcon_softc *sc, uint16_t *cols, uint16_t *rows)
1454 {
1455 	struct virtio_console_config concfg;
1456 
1457 	KASSERT(sc->vtcon_flags & VTCON_FLAG_SIZE,
1458 	    ("%s: size feature not negotiated", __func__));
1459 
1460 	vtcon_read_config(sc, &concfg);
1461 
1462 	*cols = concfg.cols;
1463 	*rows = concfg.rows;
1464 }
1465 
1466 static void
1467 vtcon_enable_interrupts(struct vtcon_softc *sc)
1468 {
1469 	struct vtcon_softc_port *scport;
1470 	struct vtcon_port *port;
1471 	int i;
1472 
1473 	VTCON_LOCK(sc);
1474 
1475 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
1476 		virtqueue_enable_intr(sc->vtcon_ctrl_rxvq);
1477 
1478 	for (i = 0; i < sc->vtcon_max_ports; i++) {
1479 		scport = &sc->vtcon_ports[i];
1480 
1481 		port = scport->vcsp_port;
1482 		if (port == NULL)
1483 			continue;
1484 
1485 		VTCON_PORT_LOCK(port);
1486 		vtcon_port_enable_intr(port);
1487 		VTCON_PORT_UNLOCK(port);
1488 	}
1489 
1490 	VTCON_UNLOCK(sc);
1491 }
1492 
1493 static void
1494 vtcon_disable_interrupts(struct vtcon_softc *sc)
1495 {
1496 	struct vtcon_softc_port *scport;
1497 	struct vtcon_port *port;
1498 	int i;
1499 
1500 	VTCON_LOCK_ASSERT(sc);
1501 
1502 	if (sc->vtcon_flags & VTCON_FLAG_MULTIPORT)
1503 		virtqueue_disable_intr(sc->vtcon_ctrl_rxvq);
1504 
1505 	for (i = 0; i < sc->vtcon_max_ports; i++) {
1506 		scport = &sc->vtcon_ports[i];
1507 
1508 		port = scport->vcsp_port;
1509 		if (port == NULL)
1510 			continue;
1511 
1512 		VTCON_PORT_LOCK(port);
1513 		vtcon_port_disable_intr(port);
1514 		VTCON_PORT_UNLOCK(port);
1515 	}
1516 }
1517