xref: /freebsd/usr.sbin/bhyve/uart_emul.c (revision b3e7694832e81d7a904a10f525f8797b753bf0d3)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2012 NetApp, Inc.
5  * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include <sys/types.h>
34 #include <dev/ic/ns16550.h>
35 #ifndef WITHOUT_CAPSICUM
36 #include <sys/capsicum.h>
37 #include <capsicum_helpers.h>
38 #endif
39 
40 #include <machine/vmm_snapshot.h>
41 
42 #include <stdio.h>
43 #include <stdlib.h>
44 #include <assert.h>
45 #include <err.h>
46 #include <errno.h>
47 #include <fcntl.h>
48 #include <termios.h>
49 #include <unistd.h>
50 #include <stdbool.h>
51 #include <string.h>
52 #include <pthread.h>
53 #include <sysexits.h>
54 
55 #include "mevent.h"
56 #include "uart_emul.h"
57 #include "debug.h"
58 
59 #define	COM1_BASE      	0x3F8
60 #define	COM1_IRQ	4
61 #define	COM2_BASE      	0x2F8
62 #define	COM2_IRQ	3
63 #define	COM3_BASE	0x3E8
64 #define	COM3_IRQ	4
65 #define	COM4_BASE	0x2E8
66 #define	COM4_IRQ	3
67 
68 #define	DEFAULT_RCLK	1843200
69 #define	DEFAULT_BAUD	9600
70 
71 #define	FCR_RX_MASK	0xC0
72 
73 #define	MCR_OUT1	0x04
74 #define	MCR_OUT2	0x08
75 
76 #define	MSR_DELTA_MASK	0x0f
77 
78 #ifndef REG_SCR
79 #define	REG_SCR		com_scr
80 #endif
81 
82 #define	FIFOSZ	16
83 
84 static bool uart_stdio;		/* stdio in use for i/o */
85 static struct termios tio_stdio_orig;
86 
87 static struct {
88 	int	baseaddr;
89 	int	irq;
90 	bool	inuse;
91 } uart_lres[] = {
92 	{ COM1_BASE, COM1_IRQ, false},
93 	{ COM2_BASE, COM2_IRQ, false},
94 	{ COM3_BASE, COM3_IRQ, false},
95 	{ COM4_BASE, COM4_IRQ, false},
96 };
97 
98 #define	UART_NLDEVS	(sizeof(uart_lres) / sizeof(uart_lres[0]))
99 
100 struct fifo {
101 	uint8_t	buf[FIFOSZ];
102 	int	rindex;		/* index to read from */
103 	int	windex;		/* index to write to */
104 	int	num;		/* number of characters in the fifo */
105 	int	size;		/* size of the fifo */
106 };
107 
108 struct ttyfd {
109 	bool	opened;
110 	int	rfd;		/* fd for reading */
111 	int	wfd;		/* fd for writing, may be == rfd */
112 };
113 
114 struct uart_softc {
115 	pthread_mutex_t mtx;	/* protects all softc elements */
116 	uint8_t	data;		/* Data register (R/W) */
117 	uint8_t ier;		/* Interrupt enable register (R/W) */
118 	uint8_t lcr;		/* Line control register (R/W) */
119 	uint8_t mcr;		/* Modem control register (R/W) */
120 	uint8_t lsr;		/* Line status register (R/W) */
121 	uint8_t msr;		/* Modem status register (R/W) */
122 	uint8_t fcr;		/* FIFO control register (W) */
123 	uint8_t scr;		/* Scratch register (R/W) */
124 
125 	uint8_t dll;		/* Baudrate divisor latch LSB */
126 	uint8_t dlh;		/* Baudrate divisor latch MSB */
127 
128 	struct fifo rxfifo;
129 	struct mevent *mev;
130 
131 	struct ttyfd tty;
132 	bool	thre_int_pending;	/* THRE interrupt pending */
133 
134 	void	*arg;
135 	uart_intr_func_t intr_assert;
136 	uart_intr_func_t intr_deassert;
137 };
138 
139 static void uart_drain(int fd, enum ev_type ev, void *arg);
140 
141 static void
142 ttyclose(void)
143 {
144 
145 	tcsetattr(STDIN_FILENO, TCSANOW, &tio_stdio_orig);
146 }
147 
148 static void
149 ttyopen(struct ttyfd *tf)
150 {
151 	struct termios orig, new;
152 
153 	tcgetattr(tf->rfd, &orig);
154 	new = orig;
155 	cfmakeraw(&new);
156 	new.c_cflag |= CLOCAL;
157 	tcsetattr(tf->rfd, TCSANOW, &new);
158 	if (uart_stdio) {
159 		tio_stdio_orig = orig;
160 		atexit(ttyclose);
161 	}
162 	raw_stdio = 1;
163 }
164 
165 static int
166 ttyread(struct ttyfd *tf)
167 {
168 	unsigned char rb;
169 
170 	if (read(tf->rfd, &rb, 1) == 1)
171 		return (rb);
172 	else
173 		return (-1);
174 }
175 
176 static void
177 ttywrite(struct ttyfd *tf, unsigned char wb)
178 {
179 
180 	(void)write(tf->wfd, &wb, 1);
181 }
182 
183 static void
184 rxfifo_reset(struct uart_softc *sc, int size)
185 {
186 	char flushbuf[32];
187 	struct fifo *fifo;
188 	ssize_t nread;
189 	int error;
190 
191 	fifo = &sc->rxfifo;
192 	bzero(fifo, sizeof(struct fifo));
193 	fifo->size = size;
194 
195 	if (sc->tty.opened) {
196 		/*
197 		 * Flush any unread input from the tty buffer.
198 		 */
199 		while (1) {
200 			nread = read(sc->tty.rfd, flushbuf, sizeof(flushbuf));
201 			if (nread != sizeof(flushbuf))
202 				break;
203 		}
204 
205 		/*
206 		 * Enable mevent to trigger when new characters are available
207 		 * on the tty fd.
208 		 */
209 		error = mevent_enable(sc->mev);
210 		assert(error == 0);
211 	}
212 }
213 
214 static int
215 rxfifo_available(struct uart_softc *sc)
216 {
217 	struct fifo *fifo;
218 
219 	fifo = &sc->rxfifo;
220 	return (fifo->num < fifo->size);
221 }
222 
223 static int
224 rxfifo_putchar(struct uart_softc *sc, uint8_t ch)
225 {
226 	struct fifo *fifo;
227 	int error;
228 
229 	fifo = &sc->rxfifo;
230 
231 	if (fifo->num < fifo->size) {
232 		fifo->buf[fifo->windex] = ch;
233 		fifo->windex = (fifo->windex + 1) % fifo->size;
234 		fifo->num++;
235 		if (!rxfifo_available(sc)) {
236 			if (sc->tty.opened) {
237 				/*
238 				 * Disable mevent callback if the FIFO is full.
239 				 */
240 				error = mevent_disable(sc->mev);
241 				assert(error == 0);
242 			}
243 		}
244 		return (0);
245 	} else
246 		return (-1);
247 }
248 
249 static int
250 rxfifo_getchar(struct uart_softc *sc)
251 {
252 	struct fifo *fifo;
253 	int c, error, wasfull;
254 
255 	wasfull = 0;
256 	fifo = &sc->rxfifo;
257 	if (fifo->num > 0) {
258 		if (!rxfifo_available(sc))
259 			wasfull = 1;
260 		c = fifo->buf[fifo->rindex];
261 		fifo->rindex = (fifo->rindex + 1) % fifo->size;
262 		fifo->num--;
263 		if (wasfull) {
264 			if (sc->tty.opened) {
265 				error = mevent_enable(sc->mev);
266 				assert(error == 0);
267 			}
268 		}
269 		return (c);
270 	} else
271 		return (-1);
272 }
273 
274 static int
275 rxfifo_numchars(struct uart_softc *sc)
276 {
277 	struct fifo *fifo = &sc->rxfifo;
278 
279 	return (fifo->num);
280 }
281 
282 static void
283 uart_opentty(struct uart_softc *sc)
284 {
285 
286 	ttyopen(&sc->tty);
287 	sc->mev = mevent_add(sc->tty.rfd, EVF_READ, uart_drain, sc);
288 	assert(sc->mev != NULL);
289 }
290 
291 static uint8_t
292 modem_status(uint8_t mcr)
293 {
294 	uint8_t msr;
295 
296 	if (mcr & MCR_LOOPBACK) {
297 		/*
298 		 * In the loopback mode certain bits from the MCR are
299 		 * reflected back into MSR.
300 		 */
301 		msr = 0;
302 		if (mcr & MCR_RTS)
303 			msr |= MSR_CTS;
304 		if (mcr & MCR_DTR)
305 			msr |= MSR_DSR;
306 		if (mcr & MCR_OUT1)
307 			msr |= MSR_RI;
308 		if (mcr & MCR_OUT2)
309 			msr |= MSR_DCD;
310 	} else {
311 		/*
312 		 * Always assert DCD and DSR so tty open doesn't block
313 		 * even if CLOCAL is turned off.
314 		 */
315 		msr = MSR_DCD | MSR_DSR;
316 	}
317 	assert((msr & MSR_DELTA_MASK) == 0);
318 
319 	return (msr);
320 }
321 
322 /*
323  * The IIR returns a prioritized interrupt reason:
324  * - receive data available
325  * - transmit holding register empty
326  * - modem status change
327  *
328  * Return an interrupt reason if one is available.
329  */
330 static int
331 uart_intr_reason(struct uart_softc *sc)
332 {
333 
334 	if ((sc->lsr & LSR_OE) != 0 && (sc->ier & IER_ERLS) != 0)
335 		return (IIR_RLS);
336 	else if (rxfifo_numchars(sc) > 0 && (sc->ier & IER_ERXRDY) != 0)
337 		return (IIR_RXTOUT);
338 	else if (sc->thre_int_pending && (sc->ier & IER_ETXRDY) != 0)
339 		return (IIR_TXRDY);
340 	else if ((sc->msr & MSR_DELTA_MASK) != 0 && (sc->ier & IER_EMSC) != 0)
341 		return (IIR_MLSC);
342 	else
343 		return (IIR_NOPEND);
344 }
345 
346 static void
347 uart_reset(struct uart_softc *sc)
348 {
349 	uint16_t divisor;
350 
351 	divisor = DEFAULT_RCLK / DEFAULT_BAUD / 16;
352 	sc->dll = divisor;
353 	sc->dlh = divisor >> 16;
354 	sc->msr = modem_status(sc->mcr);
355 
356 	rxfifo_reset(sc, 1);	/* no fifo until enabled by software */
357 }
358 
359 /*
360  * Toggle the COM port's intr pin depending on whether or not we have an
361  * interrupt condition to report to the processor.
362  */
363 static void
364 uart_toggle_intr(struct uart_softc *sc)
365 {
366 	uint8_t intr_reason;
367 
368 	intr_reason = uart_intr_reason(sc);
369 
370 	if (intr_reason == IIR_NOPEND)
371 		(*sc->intr_deassert)(sc->arg);
372 	else
373 		(*sc->intr_assert)(sc->arg);
374 }
375 
376 static void
377 uart_drain(int fd, enum ev_type ev, void *arg)
378 {
379 	struct uart_softc *sc;
380 	int ch;
381 
382 	sc = arg;
383 
384 	assert(fd == sc->tty.rfd);
385 	assert(ev == EVF_READ);
386 
387 	/*
388 	 * This routine is called in the context of the mevent thread
389 	 * to take out the softc lock to protect against concurrent
390 	 * access from a vCPU i/o exit
391 	 */
392 	pthread_mutex_lock(&sc->mtx);
393 
394 	if ((sc->mcr & MCR_LOOPBACK) != 0) {
395 		(void) ttyread(&sc->tty);
396 	} else {
397 		while (rxfifo_available(sc) &&
398 		       ((ch = ttyread(&sc->tty)) != -1)) {
399 			rxfifo_putchar(sc, ch);
400 		}
401 		uart_toggle_intr(sc);
402 	}
403 
404 	pthread_mutex_unlock(&sc->mtx);
405 }
406 
407 void
408 uart_write(struct uart_softc *sc, int offset, uint8_t value)
409 {
410 	int fifosz;
411 	uint8_t msr;
412 
413 	pthread_mutex_lock(&sc->mtx);
414 
415 	/*
416 	 * Take care of the special case DLAB accesses first
417 	 */
418 	if ((sc->lcr & LCR_DLAB) != 0) {
419 		if (offset == REG_DLL) {
420 			sc->dll = value;
421 			goto done;
422 		}
423 
424 		if (offset == REG_DLH) {
425 			sc->dlh = value;
426 			goto done;
427 		}
428 	}
429 
430         switch (offset) {
431 	case REG_DATA:
432 		if (sc->mcr & MCR_LOOPBACK) {
433 			if (rxfifo_putchar(sc, value) != 0)
434 				sc->lsr |= LSR_OE;
435 		} else if (sc->tty.opened) {
436 			ttywrite(&sc->tty, value);
437 		} /* else drop on floor */
438 		sc->thre_int_pending = true;
439 		break;
440 	case REG_IER:
441 		/* Set pending when IER_ETXRDY is raised (edge-triggered). */
442 		if ((sc->ier & IER_ETXRDY) == 0 && (value & IER_ETXRDY) != 0)
443 			sc->thre_int_pending = true;
444 		/*
445 		 * Apply mask so that bits 4-7 are 0
446 		 * Also enables bits 0-3 only if they're 1
447 		 */
448 		sc->ier = value & 0x0F;
449 		break;
450 	case REG_FCR:
451 		/*
452 		 * When moving from FIFO and 16450 mode and vice versa,
453 		 * the FIFO contents are reset.
454 		 */
455 		if ((sc->fcr & FCR_ENABLE) ^ (value & FCR_ENABLE)) {
456 			fifosz = (value & FCR_ENABLE) ? FIFOSZ : 1;
457 			rxfifo_reset(sc, fifosz);
458 		}
459 
460 		/*
461 		 * The FCR_ENABLE bit must be '1' for the programming
462 		 * of other FCR bits to be effective.
463 		 */
464 		if ((value & FCR_ENABLE) == 0) {
465 			sc->fcr = 0;
466 		} else {
467 			if ((value & FCR_RCV_RST) != 0)
468 				rxfifo_reset(sc, FIFOSZ);
469 
470 			sc->fcr = value &
471 				 (FCR_ENABLE | FCR_DMA | FCR_RX_MASK);
472 		}
473 		break;
474 	case REG_LCR:
475 		sc->lcr = value;
476 		break;
477 	case REG_MCR:
478 		/* Apply mask so that bits 5-7 are 0 */
479 		sc->mcr = value & 0x1F;
480 		msr = modem_status(sc->mcr);
481 
482 		/*
483 		 * Detect if there has been any change between the
484 		 * previous and the new value of MSR. If there is
485 		 * then assert the appropriate MSR delta bit.
486 		 */
487 		if ((msr & MSR_CTS) ^ (sc->msr & MSR_CTS))
488 			sc->msr |= MSR_DCTS;
489 		if ((msr & MSR_DSR) ^ (sc->msr & MSR_DSR))
490 			sc->msr |= MSR_DDSR;
491 		if ((msr & MSR_DCD) ^ (sc->msr & MSR_DCD))
492 			sc->msr |= MSR_DDCD;
493 		if ((sc->msr & MSR_RI) != 0 && (msr & MSR_RI) == 0)
494 			sc->msr |= MSR_TERI;
495 
496 		/*
497 		 * Update the value of MSR while retaining the delta
498 		 * bits.
499 		 */
500 		sc->msr &= MSR_DELTA_MASK;
501 		sc->msr |= msr;
502 		break;
503 	case REG_LSR:
504 		/*
505 		 * Line status register is not meant to be written to
506 		 * during normal operation.
507 		 */
508 		break;
509 	case REG_MSR:
510 		/*
511 		 * As far as I can tell MSR is a read-only register.
512 		 */
513 		break;
514 	case REG_SCR:
515 		sc->scr = value;
516 		break;
517 	default:
518 		break;
519 	}
520 
521 done:
522 	uart_toggle_intr(sc);
523 	pthread_mutex_unlock(&sc->mtx);
524 }
525 
526 uint8_t
527 uart_read(struct uart_softc *sc, int offset)
528 {
529 	uint8_t iir, intr_reason, reg;
530 
531 	pthread_mutex_lock(&sc->mtx);
532 
533 	/*
534 	 * Take care of the special case DLAB accesses first
535 	 */
536 	if ((sc->lcr & LCR_DLAB) != 0) {
537 		if (offset == REG_DLL) {
538 			reg = sc->dll;
539 			goto done;
540 		}
541 
542 		if (offset == REG_DLH) {
543 			reg = sc->dlh;
544 			goto done;
545 		}
546 	}
547 
548 	switch (offset) {
549 	case REG_DATA:
550 		reg = rxfifo_getchar(sc);
551 		break;
552 	case REG_IER:
553 		reg = sc->ier;
554 		break;
555 	case REG_IIR:
556 		iir = (sc->fcr & FCR_ENABLE) ? IIR_FIFO_MASK : 0;
557 
558 		intr_reason = uart_intr_reason(sc);
559 
560 		/*
561 		 * Deal with side effects of reading the IIR register
562 		 */
563 		if (intr_reason == IIR_TXRDY)
564 			sc->thre_int_pending = false;
565 
566 		iir |= intr_reason;
567 
568 		reg = iir;
569 		break;
570 	case REG_LCR:
571 		reg = sc->lcr;
572 		break;
573 	case REG_MCR:
574 		reg = sc->mcr;
575 		break;
576 	case REG_LSR:
577 		/* Transmitter is always ready for more data */
578 		sc->lsr |= LSR_TEMT | LSR_THRE;
579 
580 		/* Check for new receive data */
581 		if (rxfifo_numchars(sc) > 0)
582 			sc->lsr |= LSR_RXRDY;
583 		else
584 			sc->lsr &= ~LSR_RXRDY;
585 
586 		reg = sc->lsr;
587 
588 		/* The LSR_OE bit is cleared on LSR read */
589 		sc->lsr &= ~LSR_OE;
590 		break;
591 	case REG_MSR:
592 		/*
593 		 * MSR delta bits are cleared on read
594 		 */
595 		reg = sc->msr;
596 		sc->msr &= ~MSR_DELTA_MASK;
597 		break;
598 	case REG_SCR:
599 		reg = sc->scr;
600 		break;
601 	default:
602 		reg = 0xFF;
603 		break;
604 	}
605 
606 done:
607 	uart_toggle_intr(sc);
608 	pthread_mutex_unlock(&sc->mtx);
609 
610 	return (reg);
611 }
612 
613 int
614 uart_legacy_alloc(int which, int *baseaddr, int *irq)
615 {
616 
617 	if (which < 0 || which >= (int)UART_NLDEVS || uart_lres[which].inuse)
618 		return (-1);
619 
620 	uart_lres[which].inuse = true;
621 	*baseaddr = uart_lres[which].baseaddr;
622 	*irq = uart_lres[which].irq;
623 
624 	return (0);
625 }
626 
627 struct uart_softc *
628 uart_init(uart_intr_func_t intr_assert, uart_intr_func_t intr_deassert,
629     void *arg)
630 {
631 	struct uart_softc *sc;
632 
633 	sc = calloc(1, sizeof(struct uart_softc));
634 
635 	sc->arg = arg;
636 	sc->intr_assert = intr_assert;
637 	sc->intr_deassert = intr_deassert;
638 
639 	pthread_mutex_init(&sc->mtx, NULL);
640 
641 	uart_reset(sc);
642 
643 	return (sc);
644 }
645 
646 static int
647 uart_stdio_backend(struct uart_softc *sc)
648 {
649 #ifndef WITHOUT_CAPSICUM
650 	cap_rights_t rights;
651 	cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ };
652 #endif
653 
654 	if (uart_stdio)
655 		return (-1);
656 
657 	sc->tty.rfd = STDIN_FILENO;
658 	sc->tty.wfd = STDOUT_FILENO;
659 	sc->tty.opened = true;
660 
661 	if (fcntl(sc->tty.rfd, F_SETFL, O_NONBLOCK) != 0)
662 		return (-1);
663 	if (fcntl(sc->tty.wfd, F_SETFL, O_NONBLOCK) != 0)
664 		return (-1);
665 
666 #ifndef WITHOUT_CAPSICUM
667 	cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ);
668 	if (caph_rights_limit(sc->tty.rfd, &rights) == -1)
669 		errx(EX_OSERR, "Unable to apply rights for sandbox");
670 	if (caph_ioctls_limit(sc->tty.rfd, cmds, nitems(cmds)) == -1)
671 		errx(EX_OSERR, "Unable to apply rights for sandbox");
672 #endif
673 
674 	uart_stdio = true;
675 
676 	return (0);
677 }
678 
679 static int
680 uart_tty_backend(struct uart_softc *sc, const char *path)
681 {
682 #ifndef WITHOUT_CAPSICUM
683 	cap_rights_t rights;
684 	cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ };
685 #endif
686 	int fd;
687 
688 	fd = open(path, O_RDWR | O_NONBLOCK);
689 	if (fd < 0)
690 		return (-1);
691 
692 	if (!isatty(fd)) {
693 		close(fd);
694 		return (-1);
695 	}
696 
697 	sc->tty.rfd = sc->tty.wfd = fd;
698 	sc->tty.opened = true;
699 
700 #ifndef WITHOUT_CAPSICUM
701 	cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ, CAP_WRITE);
702 	if (caph_rights_limit(fd, &rights) == -1)
703 		errx(EX_OSERR, "Unable to apply rights for sandbox");
704 	if (caph_ioctls_limit(fd, cmds, nitems(cmds)) == -1)
705 		errx(EX_OSERR, "Unable to apply rights for sandbox");
706 #endif
707 
708 	return (0);
709 }
710 
711 int
712 uart_set_backend(struct uart_softc *sc, const char *device)
713 {
714 	int retval;
715 
716 	if (device == NULL)
717 		return (0);
718 
719 	if (strcmp("stdio", device) == 0)
720 		retval = uart_stdio_backend(sc);
721 	else
722 		retval = uart_tty_backend(sc, device);
723 	if (retval == 0)
724 		uart_opentty(sc);
725 
726 	return (retval);
727 }
728 
729 #ifdef BHYVE_SNAPSHOT
730 int
731 uart_snapshot(struct uart_softc *sc, struct vm_snapshot_meta *meta)
732 {
733 	int ret;
734 
735 	SNAPSHOT_VAR_OR_LEAVE(sc->data, meta, ret, done);
736 	SNAPSHOT_VAR_OR_LEAVE(sc->ier, meta, ret, done);
737 	SNAPSHOT_VAR_OR_LEAVE(sc->lcr, meta, ret, done);
738 	SNAPSHOT_VAR_OR_LEAVE(sc->mcr, meta, ret, done);
739 	SNAPSHOT_VAR_OR_LEAVE(sc->lsr, meta, ret, done);
740 	SNAPSHOT_VAR_OR_LEAVE(sc->msr, meta, ret, done);
741 	SNAPSHOT_VAR_OR_LEAVE(sc->fcr, meta, ret, done);
742 	SNAPSHOT_VAR_OR_LEAVE(sc->scr, meta, ret, done);
743 
744 	SNAPSHOT_VAR_OR_LEAVE(sc->dll, meta, ret, done);
745 	SNAPSHOT_VAR_OR_LEAVE(sc->dlh, meta, ret, done);
746 
747 	SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.rindex, meta, ret, done);
748 	SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.windex, meta, ret, done);
749 	SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.num, meta, ret, done);
750 	SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.size, meta, ret, done);
751 	SNAPSHOT_BUF_OR_LEAVE(sc->rxfifo.buf, sizeof(sc->rxfifo.buf),
752 			      meta, ret, done);
753 
754 	sc->thre_int_pending = 1;
755 
756 done:
757 	return (ret);
758 }
759 #endif
760