xref: /illumos-gate/usr/src/test/libc-tests/tests/isatty.c (revision 763f1f5f97e4c16840af2ced98915f0ed0f46616)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2022 Oxide Computer Company
14  */
15 
16 /*
17  * This program verifies that isatty(3C) correctly handles and sets errno for
18  * different cases.
19  */
20 
21 #include <sys/types.h>
22 #include <sys/stat.h>
23 #include <fcntl.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <limits.h>
27 #include <errno.h>
28 #include <err.h>
29 #include <stdbool.h>
30 #include <sys/stropts.h>
31 #include <sys/sysmacros.h>
32 
33 #include "common/openpty.c"
34 
35 int
36 main(void)
37 {
38 	int sfd, mfd;
39 	int ret = EXIT_SUCCESS;
40 	const int badfds[] = { 3, -1, INT_MAX, INT_MIN, 7777 };
41 	const char *notttys[] = { "/proc/self/status", "/usr/lib/64/libc.so.1",
42 	    "/dev/zero", "/dev/tcp", "/dev/poll", "/etc/default/init" };
43 
44 	/*
45 	 * We start off by using closefrom() to verify that we don't have
46 	 * anything open other than the standard file descriptors, allowing us
47 	 * to pick FDs that make sense.
48 	 */
49 	closefrom(STDERR_FILENO + 1);
50 
51 	for (size_t i = 0; i < ARRAY_SIZE(badfds); i++) {
52 		/*
53 		 * We explicitly clear errno to prove that we are setting it.
54 		 * The closefrom() will hit EBADF and we want to clear that out
55 		 * from the test (as well as any side effects below.
56 		 */
57 		errno = 0;
58 		if (isatty(badfds[i]) != 0) {
59 			warnx("TEST FAILED: isatty(%d) returned success on bad "
60 			    "fd", badfds[i]);
61 			ret = EXIT_FAILURE;
62 			continue;
63 		}
64 
65 		if (errno != EBADF) {
66 			int e = errno;
67 			warnx("TEST FAILED: isatty(%d) didn't set EBADF, "
68 			    "found: %d", badfds[i], e);
69 			ret = EXIT_FAILURE;
70 			continue;
71 		}
72 
73 		(void) printf("TEST PASSED: isatty(%d) failed with EBADF\n",
74 		    badfds[i]);
75 	}
76 
77 	for (size_t i = 0; i < ARRAY_SIZE(notttys); i++) {
78 		int fd = open(notttys[i], O_RDONLY);
79 		int ttyret, ttyerr;
80 
81 		if (fd < 0) {
82 			warn("TEST FAILED: failed to open %s", notttys[i]);
83 			ret = EXIT_FAILURE;
84 			continue;
85 		}
86 
87 		errno = 0;
88 		ttyret = isatty(fd);
89 		ttyerr = errno;
90 		(void) close(fd);
91 
92 		if (ttyret != 0) {
93 			warnx("TEST FAILED: %s is somehow a tty!", notttys[i]);
94 			ret = EXIT_FAILURE;
95 			continue;
96 		}
97 
98 		if (ttyerr != ENOTTY) {
99 			warnx("TEST FAILED: got wrong errno for %s, expected "
100 			    "ENOTTY, found %d", notttys[i], ttyerr);
101 			ret = EXIT_FAILURE;
102 			continue;
103 		}
104 
105 		(void) printf("TEST PASSED: %s is not a tty, errno was "
106 		    "ENOTTY\n", notttys[i]);
107 	}
108 
109 	if (!openpty(&mfd, &sfd)) {
110 		errx(EXIT_FAILURE, "TEST FAILED: failed to open a pty");
111 	}
112 
113 	if (isatty(sfd) != 1) {
114 		warnx("subsidiary PTY fd somehow isn't a TTY!");
115 		ret = EXIT_FAILURE;
116 	} else {
117 		(void) printf("TEST PASSED: subsidiary PTY is a TTY\n");
118 	}
119 
120 	if (isatty(mfd) != 0) {
121 		warnx("manager PTY fd somehow is a TTY!");
122 		ret = EXIT_FAILURE;
123 	} else {
124 		(void) printf("TEST PASSED: manager PTY is not a TTY\n");
125 	}
126 
127 	(void) close(mfd);
128 	(void) close(sfd);
129 
130 	return (ret);
131 }
132