xref: /freebsd/usr.sbin/bhyve/uart_emul.c (revision 3cbb4cc200f8a0ad7ed08233425ea54524a21f1c)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012 NetApp, Inc.
5  * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  * $FreeBSD$
30  */
31 
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34 
35 #include <sys/types.h>
36 #include <dev/ic/ns16550.h>
37 #ifndef WITHOUT_CAPSICUM
38 #include <sys/capsicum.h>
39 #include <capsicum_helpers.h>
40 #endif
41 
42 #include <machine/vmm_snapshot.h>
43 
44 #include <stdio.h>
45 #include <stdlib.h>
46 #include <assert.h>
47 #include <err.h>
48 #include <errno.h>
49 #include <fcntl.h>
50 #include <termios.h>
51 #include <unistd.h>
52 #include <stdbool.h>
53 #include <string.h>
54 #include <pthread.h>
55 #include <sysexits.h>
56 
57 #include "mevent.h"
58 #include "uart_emul.h"
59 #include "debug.h"
60 
61 #define	COM1_BASE      	0x3F8
62 #define	COM1_IRQ	4
63 #define	COM2_BASE      	0x2F8
64 #define	COM2_IRQ	3
65 
66 #define	DEFAULT_RCLK	1843200
67 #define	DEFAULT_BAUD	9600
68 
69 #define	FCR_RX_MASK	0xC0
70 
71 #define	MCR_OUT1	0x04
72 #define	MCR_OUT2	0x08
73 
74 #define	MSR_DELTA_MASK	0x0f
75 
76 #ifndef REG_SCR
77 #define	REG_SCR		com_scr
78 #endif
79 
80 #define	FIFOSZ	16
81 
82 static bool uart_stdio;		/* stdio in use for i/o */
83 static struct termios tio_stdio_orig;
84 
85 static struct {
86 	int	baseaddr;
87 	int	irq;
88 	bool	inuse;
89 } uart_lres[] = {
90 	{ COM1_BASE, COM1_IRQ, false},
91 	{ COM2_BASE, COM2_IRQ, false},
92 };
93 
94 #define	UART_NLDEVS	(sizeof(uart_lres) / sizeof(uart_lres[0]))
95 
96 struct fifo {
97 	uint8_t	buf[FIFOSZ];
98 	int	rindex;		/* index to read from */
99 	int	windex;		/* index to write to */
100 	int	num;		/* number of characters in the fifo */
101 	int	size;		/* size of the fifo */
102 };
103 
104 struct ttyfd {
105 	bool	opened;
106 	int	rfd;		/* fd for reading */
107 	int	wfd;		/* fd for writing, may be == rfd */
108 };
109 
110 struct uart_softc {
111 	pthread_mutex_t mtx;	/* protects all softc elements */
112 	uint8_t	data;		/* Data register (R/W) */
113 	uint8_t ier;		/* Interrupt enable register (R/W) */
114 	uint8_t lcr;		/* Line control register (R/W) */
115 	uint8_t mcr;		/* Modem control register (R/W) */
116 	uint8_t lsr;		/* Line status register (R/W) */
117 	uint8_t msr;		/* Modem status register (R/W) */
118 	uint8_t fcr;		/* FIFO control register (W) */
119 	uint8_t scr;		/* Scratch register (R/W) */
120 
121 	uint8_t dll;		/* Baudrate divisor latch LSB */
122 	uint8_t dlh;		/* Baudrate divisor latch MSB */
123 
124 	struct fifo rxfifo;
125 	struct mevent *mev;
126 
127 	struct ttyfd tty;
128 	bool	thre_int_pending;	/* THRE interrupt pending */
129 
130 	void	*arg;
131 	uart_intr_func_t intr_assert;
132 	uart_intr_func_t intr_deassert;
133 };
134 
135 static void uart_drain(int fd, enum ev_type ev, void *arg);
136 
137 static void
138 ttyclose(void)
139 {
140 
141 	tcsetattr(STDIN_FILENO, TCSANOW, &tio_stdio_orig);
142 }
143 
144 static void
145 ttyopen(struct ttyfd *tf)
146 {
147 	struct termios orig, new;
148 
149 	tcgetattr(tf->rfd, &orig);
150 	new = orig;
151 	cfmakeraw(&new);
152 	new.c_cflag |= CLOCAL;
153 	tcsetattr(tf->rfd, TCSANOW, &new);
154 	if (uart_stdio) {
155 		tio_stdio_orig = orig;
156 		atexit(ttyclose);
157 	}
158 	raw_stdio = 1;
159 }
160 
161 static int
162 ttyread(struct ttyfd *tf)
163 {
164 	unsigned char rb;
165 
166 	if (read(tf->rfd, &rb, 1) == 1)
167 		return (rb);
168 	else
169 		return (-1);
170 }
171 
172 static void
173 ttywrite(struct ttyfd *tf, unsigned char wb)
174 {
175 
176 	(void)write(tf->wfd, &wb, 1);
177 }
178 
179 static void
180 rxfifo_reset(struct uart_softc *sc, int size)
181 {
182 	char flushbuf[32];
183 	struct fifo *fifo;
184 	ssize_t nread;
185 	int error;
186 
187 	fifo = &sc->rxfifo;
188 	bzero(fifo, sizeof(struct fifo));
189 	fifo->size = size;
190 
191 	if (sc->tty.opened) {
192 		/*
193 		 * Flush any unread input from the tty buffer.
194 		 */
195 		while (1) {
196 			nread = read(sc->tty.rfd, flushbuf, sizeof(flushbuf));
197 			if (nread != sizeof(flushbuf))
198 				break;
199 		}
200 
201 		/*
202 		 * Enable mevent to trigger when new characters are available
203 		 * on the tty fd.
204 		 */
205 		error = mevent_enable(sc->mev);
206 		assert(error == 0);
207 	}
208 }
209 
210 static int
211 rxfifo_available(struct uart_softc *sc)
212 {
213 	struct fifo *fifo;
214 
215 	fifo = &sc->rxfifo;
216 	return (fifo->num < fifo->size);
217 }
218 
219 static int
220 rxfifo_putchar(struct uart_softc *sc, uint8_t ch)
221 {
222 	struct fifo *fifo;
223 	int error;
224 
225 	fifo = &sc->rxfifo;
226 
227 	if (fifo->num < fifo->size) {
228 		fifo->buf[fifo->windex] = ch;
229 		fifo->windex = (fifo->windex + 1) % fifo->size;
230 		fifo->num++;
231 		if (!rxfifo_available(sc)) {
232 			if (sc->tty.opened) {
233 				/*
234 				 * Disable mevent callback if the FIFO is full.
235 				 */
236 				error = mevent_disable(sc->mev);
237 				assert(error == 0);
238 			}
239 		}
240 		return (0);
241 	} else
242 		return (-1);
243 }
244 
245 static int
246 rxfifo_getchar(struct uart_softc *sc)
247 {
248 	struct fifo *fifo;
249 	int c, error, wasfull;
250 
251 	wasfull = 0;
252 	fifo = &sc->rxfifo;
253 	if (fifo->num > 0) {
254 		if (!rxfifo_available(sc))
255 			wasfull = 1;
256 		c = fifo->buf[fifo->rindex];
257 		fifo->rindex = (fifo->rindex + 1) % fifo->size;
258 		fifo->num--;
259 		if (wasfull) {
260 			if (sc->tty.opened) {
261 				error = mevent_enable(sc->mev);
262 				assert(error == 0);
263 			}
264 		}
265 		return (c);
266 	} else
267 		return (-1);
268 }
269 
270 static int
271 rxfifo_numchars(struct uart_softc *sc)
272 {
273 	struct fifo *fifo = &sc->rxfifo;
274 
275 	return (fifo->num);
276 }
277 
278 static void
279 uart_opentty(struct uart_softc *sc)
280 {
281 
282 	ttyopen(&sc->tty);
283 	sc->mev = mevent_add(sc->tty.rfd, EVF_READ, uart_drain, sc);
284 	assert(sc->mev != NULL);
285 }
286 
287 static uint8_t
288 modem_status(uint8_t mcr)
289 {
290 	uint8_t msr;
291 
292 	if (mcr & MCR_LOOPBACK) {
293 		/*
294 		 * In the loopback mode certain bits from the MCR are
295 		 * reflected back into MSR.
296 		 */
297 		msr = 0;
298 		if (mcr & MCR_RTS)
299 			msr |= MSR_CTS;
300 		if (mcr & MCR_DTR)
301 			msr |= MSR_DSR;
302 		if (mcr & MCR_OUT1)
303 			msr |= MSR_RI;
304 		if (mcr & MCR_OUT2)
305 			msr |= MSR_DCD;
306 	} else {
307 		/*
308 		 * Always assert DCD and DSR so tty open doesn't block
309 		 * even if CLOCAL is turned off.
310 		 */
311 		msr = MSR_DCD | MSR_DSR;
312 	}
313 	assert((msr & MSR_DELTA_MASK) == 0);
314 
315 	return (msr);
316 }
317 
318 /*
319  * The IIR returns a prioritized interrupt reason:
320  * - receive data available
321  * - transmit holding register empty
322  * - modem status change
323  *
324  * Return an interrupt reason if one is available.
325  */
326 static int
327 uart_intr_reason(struct uart_softc *sc)
328 {
329 
330 	if ((sc->lsr & LSR_OE) != 0 && (sc->ier & IER_ERLS) != 0)
331 		return (IIR_RLS);
332 	else if (rxfifo_numchars(sc) > 0 && (sc->ier & IER_ERXRDY) != 0)
333 		return (IIR_RXTOUT);
334 	else if (sc->thre_int_pending && (sc->ier & IER_ETXRDY) != 0)
335 		return (IIR_TXRDY);
336 	else if ((sc->msr & MSR_DELTA_MASK) != 0 && (sc->ier & IER_EMSC) != 0)
337 		return (IIR_MLSC);
338 	else
339 		return (IIR_NOPEND);
340 }
341 
342 static void
343 uart_reset(struct uart_softc *sc)
344 {
345 	uint16_t divisor;
346 
347 	divisor = DEFAULT_RCLK / DEFAULT_BAUD / 16;
348 	sc->dll = divisor;
349 	sc->dlh = divisor >> 16;
350 	sc->msr = modem_status(sc->mcr);
351 
352 	rxfifo_reset(sc, 1);	/* no fifo until enabled by software */
353 }
354 
355 /*
356  * Toggle the COM port's intr pin depending on whether or not we have an
357  * interrupt condition to report to the processor.
358  */
359 static void
360 uart_toggle_intr(struct uart_softc *sc)
361 {
362 	uint8_t intr_reason;
363 
364 	intr_reason = uart_intr_reason(sc);
365 
366 	if (intr_reason == IIR_NOPEND)
367 		(*sc->intr_deassert)(sc->arg);
368 	else
369 		(*sc->intr_assert)(sc->arg);
370 }
371 
372 static void
373 uart_drain(int fd, enum ev_type ev, void *arg)
374 {
375 	struct uart_softc *sc;
376 	int ch;
377 
378 	sc = arg;
379 
380 	assert(fd == sc->tty.rfd);
381 	assert(ev == EVF_READ);
382 
383 	/*
384 	 * This routine is called in the context of the mevent thread
385 	 * to take out the softc lock to protect against concurrent
386 	 * access from a vCPU i/o exit
387 	 */
388 	pthread_mutex_lock(&sc->mtx);
389 
390 	if ((sc->mcr & MCR_LOOPBACK) != 0) {
391 		(void) ttyread(&sc->tty);
392 	} else {
393 		while (rxfifo_available(sc) &&
394 		       ((ch = ttyread(&sc->tty)) != -1)) {
395 			rxfifo_putchar(sc, ch);
396 		}
397 		uart_toggle_intr(sc);
398 	}
399 
400 	pthread_mutex_unlock(&sc->mtx);
401 }
402 
403 void
404 uart_write(struct uart_softc *sc, int offset, uint8_t value)
405 {
406 	int fifosz;
407 	uint8_t msr;
408 
409 	pthread_mutex_lock(&sc->mtx);
410 
411 	/*
412 	 * Take care of the special case DLAB accesses first
413 	 */
414 	if ((sc->lcr & LCR_DLAB) != 0) {
415 		if (offset == REG_DLL) {
416 			sc->dll = value;
417 			goto done;
418 		}
419 
420 		if (offset == REG_DLH) {
421 			sc->dlh = value;
422 			goto done;
423 		}
424 	}
425 
426         switch (offset) {
427 	case REG_DATA:
428 		if (sc->mcr & MCR_LOOPBACK) {
429 			if (rxfifo_putchar(sc, value) != 0)
430 				sc->lsr |= LSR_OE;
431 		} else if (sc->tty.opened) {
432 			ttywrite(&sc->tty, value);
433 		} /* else drop on floor */
434 		sc->thre_int_pending = true;
435 		break;
436 	case REG_IER:
437 		/* Set pending when IER_ETXRDY is raised (edge-triggered). */
438 		if ((sc->ier & IER_ETXRDY) == 0 && (value & IER_ETXRDY) != 0)
439 			sc->thre_int_pending = true;
440 		/*
441 		 * Apply mask so that bits 4-7 are 0
442 		 * Also enables bits 0-3 only if they're 1
443 		 */
444 		sc->ier = value & 0x0F;
445 		break;
446 	case REG_FCR:
447 		/*
448 		 * When moving from FIFO and 16450 mode and vice versa,
449 		 * the FIFO contents are reset.
450 		 */
451 		if ((sc->fcr & FCR_ENABLE) ^ (value & FCR_ENABLE)) {
452 			fifosz = (value & FCR_ENABLE) ? FIFOSZ : 1;
453 			rxfifo_reset(sc, fifosz);
454 		}
455 
456 		/*
457 		 * The FCR_ENABLE bit must be '1' for the programming
458 		 * of other FCR bits to be effective.
459 		 */
460 		if ((value & FCR_ENABLE) == 0) {
461 			sc->fcr = 0;
462 		} else {
463 			if ((value & FCR_RCV_RST) != 0)
464 				rxfifo_reset(sc, FIFOSZ);
465 
466 			sc->fcr = value &
467 				 (FCR_ENABLE | FCR_DMA | FCR_RX_MASK);
468 		}
469 		break;
470 	case REG_LCR:
471 		sc->lcr = value;
472 		break;
473 	case REG_MCR:
474 		/* Apply mask so that bits 5-7 are 0 */
475 		sc->mcr = value & 0x1F;
476 		msr = modem_status(sc->mcr);
477 
478 		/*
479 		 * Detect if there has been any change between the
480 		 * previous and the new value of MSR. If there is
481 		 * then assert the appropriate MSR delta bit.
482 		 */
483 		if ((msr & MSR_CTS) ^ (sc->msr & MSR_CTS))
484 			sc->msr |= MSR_DCTS;
485 		if ((msr & MSR_DSR) ^ (sc->msr & MSR_DSR))
486 			sc->msr |= MSR_DDSR;
487 		if ((msr & MSR_DCD) ^ (sc->msr & MSR_DCD))
488 			sc->msr |= MSR_DDCD;
489 		if ((sc->msr & MSR_RI) != 0 && (msr & MSR_RI) == 0)
490 			sc->msr |= MSR_TERI;
491 
492 		/*
493 		 * Update the value of MSR while retaining the delta
494 		 * bits.
495 		 */
496 		sc->msr &= MSR_DELTA_MASK;
497 		sc->msr |= msr;
498 		break;
499 	case REG_LSR:
500 		/*
501 		 * Line status register is not meant to be written to
502 		 * during normal operation.
503 		 */
504 		break;
505 	case REG_MSR:
506 		/*
507 		 * As far as I can tell MSR is a read-only register.
508 		 */
509 		break;
510 	case REG_SCR:
511 		sc->scr = value;
512 		break;
513 	default:
514 		break;
515 	}
516 
517 done:
518 	uart_toggle_intr(sc);
519 	pthread_mutex_unlock(&sc->mtx);
520 }
521 
522 uint8_t
523 uart_read(struct uart_softc *sc, int offset)
524 {
525 	uint8_t iir, intr_reason, reg;
526 
527 	pthread_mutex_lock(&sc->mtx);
528 
529 	/*
530 	 * Take care of the special case DLAB accesses first
531 	 */
532 	if ((sc->lcr & LCR_DLAB) != 0) {
533 		if (offset == REG_DLL) {
534 			reg = sc->dll;
535 			goto done;
536 		}
537 
538 		if (offset == REG_DLH) {
539 			reg = sc->dlh;
540 			goto done;
541 		}
542 	}
543 
544 	switch (offset) {
545 	case REG_DATA:
546 		reg = rxfifo_getchar(sc);
547 		break;
548 	case REG_IER:
549 		reg = sc->ier;
550 		break;
551 	case REG_IIR:
552 		iir = (sc->fcr & FCR_ENABLE) ? IIR_FIFO_MASK : 0;
553 
554 		intr_reason = uart_intr_reason(sc);
555 
556 		/*
557 		 * Deal with side effects of reading the IIR register
558 		 */
559 		if (intr_reason == IIR_TXRDY)
560 			sc->thre_int_pending = false;
561 
562 		iir |= intr_reason;
563 
564 		reg = iir;
565 		break;
566 	case REG_LCR:
567 		reg = sc->lcr;
568 		break;
569 	case REG_MCR:
570 		reg = sc->mcr;
571 		break;
572 	case REG_LSR:
573 		/* Transmitter is always ready for more data */
574 		sc->lsr |= LSR_TEMT | LSR_THRE;
575 
576 		/* Check for new receive data */
577 		if (rxfifo_numchars(sc) > 0)
578 			sc->lsr |= LSR_RXRDY;
579 		else
580 			sc->lsr &= ~LSR_RXRDY;
581 
582 		reg = sc->lsr;
583 
584 		/* The LSR_OE bit is cleared on LSR read */
585 		sc->lsr &= ~LSR_OE;
586 		break;
587 	case REG_MSR:
588 		/*
589 		 * MSR delta bits are cleared on read
590 		 */
591 		reg = sc->msr;
592 		sc->msr &= ~MSR_DELTA_MASK;
593 		break;
594 	case REG_SCR:
595 		reg = sc->scr;
596 		break;
597 	default:
598 		reg = 0xFF;
599 		break;
600 	}
601 
602 done:
603 	uart_toggle_intr(sc);
604 	pthread_mutex_unlock(&sc->mtx);
605 
606 	return (reg);
607 }
608 
609 int
610 uart_legacy_alloc(int which, int *baseaddr, int *irq)
611 {
612 
613 	if (which < 0 || which >= UART_NLDEVS || uart_lres[which].inuse)
614 		return (-1);
615 
616 	uart_lres[which].inuse = true;
617 	*baseaddr = uart_lres[which].baseaddr;
618 	*irq = uart_lres[which].irq;
619 
620 	return (0);
621 }
622 
623 struct uart_softc *
624 uart_init(uart_intr_func_t intr_assert, uart_intr_func_t intr_deassert,
625     void *arg)
626 {
627 	struct uart_softc *sc;
628 
629 	sc = calloc(1, sizeof(struct uart_softc));
630 
631 	sc->arg = arg;
632 	sc->intr_assert = intr_assert;
633 	sc->intr_deassert = intr_deassert;
634 
635 	pthread_mutex_init(&sc->mtx, NULL);
636 
637 	uart_reset(sc);
638 
639 	return (sc);
640 }
641 
642 static int
643 uart_stdio_backend(struct uart_softc *sc)
644 {
645 #ifndef WITHOUT_CAPSICUM
646 	cap_rights_t rights;
647 	cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ };
648 #endif
649 
650 	if (uart_stdio)
651 		return (-1);
652 
653 	sc->tty.rfd = STDIN_FILENO;
654 	sc->tty.wfd = STDOUT_FILENO;
655 	sc->tty.opened = true;
656 
657 	if (fcntl(sc->tty.rfd, F_SETFL, O_NONBLOCK) != 0)
658 		return (-1);
659 	if (fcntl(sc->tty.wfd, F_SETFL, O_NONBLOCK) != 0)
660 		return (-1);
661 
662 #ifndef WITHOUT_CAPSICUM
663 	cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ);
664 	if (caph_rights_limit(sc->tty.rfd, &rights) == -1)
665 		errx(EX_OSERR, "Unable to apply rights for sandbox");
666 	if (caph_ioctls_limit(sc->tty.rfd, cmds, nitems(cmds)) == -1)
667 		errx(EX_OSERR, "Unable to apply rights for sandbox");
668 #endif
669 
670 	uart_stdio = true;
671 
672 	return (0);
673 }
674 
675 static int
676 uart_tty_backend(struct uart_softc *sc, const char *opts)
677 {
678 #ifndef WITHOUT_CAPSICUM
679 	cap_rights_t rights;
680 	cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ };
681 #endif
682 	int fd;
683 
684 	fd = open(opts, O_RDWR | O_NONBLOCK);
685 	if (fd < 0)
686 		return (-1);
687 
688 	if (!isatty(fd)) {
689 		close(fd);
690 		return (-1);
691 	}
692 
693 	sc->tty.rfd = sc->tty.wfd = fd;
694 	sc->tty.opened = true;
695 
696 #ifndef WITHOUT_CAPSICUM
697 	cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ, CAP_WRITE);
698 	if (caph_rights_limit(fd, &rights) == -1)
699 		errx(EX_OSERR, "Unable to apply rights for sandbox");
700 	if (caph_ioctls_limit(fd, cmds, nitems(cmds)) == -1)
701 		errx(EX_OSERR, "Unable to apply rights for sandbox");
702 #endif
703 
704 	return (0);
705 }
706 
707 int
708 uart_set_backend(struct uart_softc *sc, const char *opts)
709 {
710 	int retval;
711 
712 	if (opts == NULL)
713 		return (0);
714 
715 	if (strcmp("stdio", opts) == 0)
716 		retval = uart_stdio_backend(sc);
717 	else
718 		retval = uart_tty_backend(sc, opts);
719 	if (retval == 0)
720 		uart_opentty(sc);
721 
722 	return (retval);
723 }
724 
725 #ifdef BHYVE_SNAPSHOT
726 int
727 uart_snapshot(struct uart_softc *sc, struct vm_snapshot_meta *meta)
728 {
729 	int ret;
730 
731 	SNAPSHOT_VAR_OR_LEAVE(sc->data, meta, ret, done);
732 	SNAPSHOT_VAR_OR_LEAVE(sc->ier, meta, ret, done);
733 	SNAPSHOT_VAR_OR_LEAVE(sc->lcr, meta, ret, done);
734 	SNAPSHOT_VAR_OR_LEAVE(sc->mcr, meta, ret, done);
735 	SNAPSHOT_VAR_OR_LEAVE(sc->lsr, meta, ret, done);
736 	SNAPSHOT_VAR_OR_LEAVE(sc->msr, meta, ret, done);
737 	SNAPSHOT_VAR_OR_LEAVE(sc->fcr, meta, ret, done);
738 	SNAPSHOT_VAR_OR_LEAVE(sc->scr, meta, ret, done);
739 
740 	SNAPSHOT_VAR_OR_LEAVE(sc->dll, meta, ret, done);
741 	SNAPSHOT_VAR_OR_LEAVE(sc->dlh, meta, ret, done);
742 
743 	SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.rindex, meta, ret, done);
744 	SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.windex, meta, ret, done);
745 	SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.num, meta, ret, done);
746 	SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.size, meta, ret, done);
747 	SNAPSHOT_BUF_OR_LEAVE(sc->rxfifo.buf, sizeof(sc->rxfifo.buf),
748 			      meta, ret, done);
749 
750 	sc->thre_int_pending = 1;
751 
752 done:
753 	return (ret);
754 }
755 #endif
756