xref: /illumos-gate/usr/src/test/libc-tests/tests/isatty.c (revision f137b22e734e85642da3e56e8b94da3f5f027c73)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2022 Oxide Computer Company
14  */
15 
16 /*
17  * This program verifies that isatty(3C) correctly handles and sets errno for
18  * different cases.
19  */
20 
21 #include <sys/types.h>
22 #include <sys/stat.h>
23 #include <fcntl.h>
24 #include <stdlib.h>
25 #include <unistd.h>
26 #include <limits.h>
27 #include <errno.h>
28 #include <err.h>
29 #include <stdbool.h>
30 #include <sys/stropts.h>
31 #include <sys/sysmacros.h>
32 
33 /*
34  * This is named this way with the hope that it'll be replaced someday by
35  * openpty.
36  */
37 bool
38 openpty(int *mfdp, int *sfdp)
39 {
40 	int sfd;
41 	int mfd = posix_openpt(O_RDWR | O_NOCTTY);
42 	const char *name;
43 
44 	if (mfd < 0) {
45 		warn("failed to open a pseudo-terminal");
46 		return (false);
47 	}
48 
49 	if (grantpt(mfd) != 0 || unlockpt(mfd) != 0) {
50 		warn("failed to grant and unlock the manager fd");
51 		(void) close(mfd);
52 		return (false);
53 	}
54 
55 	name = ptsname(mfd);
56 	if (name == NULL) {
57 		warnx("failed to get ptsname for fd %d", mfd);
58 		(void) close(mfd);
59 		return (false);
60 	}
61 
62 	sfd = open(name, O_RDWR | O_NOCTTY);
63 	if (sfd < 0) {
64 		warn("failed to open pty %s", name);
65 		(void) close(mfd);
66 		return (false);
67 	}
68 
69 	if (ioctl(sfd, __I_PUSH_NOCTTY, "ptem") < 0 ||
70 	    ioctl(sfd, __I_PUSH_NOCTTY, "ldterm") < 0) {
71 		warn("failed to push streams modules");
72 		(void) close(mfd);
73 		(void) close(sfd);
74 	}
75 
76 	*sfdp = sfd;
77 	*mfdp = mfd;
78 	return (true);
79 }
80 
81 int
82 main(void)
83 {
84 	int sfd, mfd;
85 	int ret = EXIT_SUCCESS;
86 	const int badfds[] = { 3, -1, INT_MAX, INT_MIN, 7777 };
87 	const char *notttys[] = { "/proc/self/status", "/usr/lib/64/libc.so.1",
88 	    "/dev/zero", "/dev/tcp", "/dev/poll", "/etc/default/init" };
89 
90 	/*
91 	 * We start off by using closefrom() to verify that we don't have
92 	 * anything open other than the standard file descriptors, allowing us
93 	 * to pick FDs that make sense.
94 	 */
95 	closefrom(STDERR_FILENO + 1);
96 
97 	for (size_t i = 0; i < ARRAY_SIZE(badfds); i++) {
98 		/*
99 		 * We explicitly clear errno to prove that we are setting it.
100 		 * The closefrom() will hit EBADF and we want to clear that out
101 		 * from the test (as well as any side effects below.
102 		 */
103 		errno = 0;
104 		if (isatty(badfds[i]) != 0) {
105 			warnx("TEST FAILED: isatty(%d) returned success on bad "
106 			    "fd", badfds[i]);
107 			ret = EXIT_FAILURE;
108 			continue;
109 		}
110 
111 		if (errno != EBADF) {
112 			int e = errno;
113 			warnx("TEST FAILED: isatty(%d) didn't set EBADF, "
114 			    "found: %d", badfds[i], e);
115 			ret = EXIT_FAILURE;
116 			continue;
117 		}
118 
119 		(void) printf("TEST PASSED: isatty(%d) failed with EBADF\n",
120 		    badfds[i]);
121 	}
122 
123 	for (size_t i = 0; i < ARRAY_SIZE(notttys); i++) {
124 		int fd = open(notttys[i], O_RDONLY);
125 		int ttyret, ttyerr;
126 
127 		if (fd < 0) {
128 			warn("TEST FAILED: failed to open %s", notttys[i]);
129 			ret = EXIT_FAILURE;
130 			continue;
131 		}
132 
133 		errno = 0;
134 		ttyret = isatty(fd);
135 		ttyerr = errno;
136 		(void) close(fd);
137 
138 		if (ttyret != 0) {
139 			warnx("TEST FAILED: %s is somehow a tty!", notttys[i]);
140 			ret = EXIT_FAILURE;
141 			continue;
142 		}
143 
144 		if (ttyerr != ENOTTY) {
145 			warnx("TEST FAILED: got wrong errno for %s, expected "
146 			    "ENOTTY, found %d", notttys[i], ttyerr);
147 			ret = EXIT_FAILURE;
148 			continue;
149 		}
150 
151 		(void) printf("TEST PASSED: %s is not a tty, errno was "
152 		    "ENOTTY\n", notttys[i]);
153 	}
154 
155 	if (!openpty(&mfd, &sfd)) {
156 		errx(EXIT_FAILURE, "TEST FAILED: failed to open a pty");
157 	}
158 
159 	if (isatty(sfd) != 1) {
160 		warnx("subsidiary PTY fd somehow isn't a TTY!");
161 		ret = EXIT_FAILURE;
162 	} else {
163 		(void) printf("TEST PASSED: subsidiary PTY is a TTY\n");
164 	}
165 
166 	if (isatty(mfd) != 0) {
167 		warnx("manager PTY fd somehow is a TTY!");
168 		ret = EXIT_FAILURE;
169 	} else {
170 		(void) printf("TEST PASSED: manager PTY is not a TTY\n");
171 	}
172 
173 	(void) close(mfd);
174 	(void) close(sfd);
175 
176 	return (ret);
177 }
178