xref: /titanic_50/usr/src/uts/common/exec/aout/aout.c (revision b86efd96f8acd85ddaa930a2f0c1d664237e4aaf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/fpu/fpusystm.h>
32 #include <sys/sysmacros.h>
33 #include <sys/signal.h>
34 #include <sys/cred.h>
35 #include <sys/user.h>
36 #include <sys/errno.h>
37 #include <sys/vnode.h>
38 #include <sys/mman.h>
39 #include <sys/kmem.h>
40 #include <sys/proc.h>
41 #include <sys/pathname.h>
42 #include <sys/cmn_err.h>
43 #include <sys/debug.h>
44 #include <sys/exec.h>
45 #include <sys/exechdr.h>
46 #include <sys/auxv.h>
47 #include <sys/core.h>
48 #include <sys/vmparam.h>
49 #include <sys/archsystm.h>
50 #include <sys/fs/swapnode.h>
51 #include <sys/modctl.h>
52 #include <vm/anon.h>
53 #include <vm/as.h>
54 #include <vm/seg.h>
55 
56 static int aoutexec(vnode_t *vp, execa_t *uap, uarg_t *args,
57     intpdata_t *idatap, int level, long *execsz, int setid,
58     caddr_t exec_file, cred_t *cred, int brand_action);
59 static int get_aout_head(struct vnode **vpp, struct exdata *edp, long *execsz,
60     int *isdyn);
61 static int aoutcore(vnode_t *vp, proc_t *pp, cred_t *credp,
62     rlim64_t rlimit, int sig, core_content_t content);
63 #ifdef	_LP64
64 extern int elf32exec(vnode_t *, execa_t *, uarg_t *, intpdata_t *, int,
65     long *, int, caddr_t, cred_t *, int);
66 extern int elf32core(vnode_t *, proc_t *, cred_t *, rlim64_t, int,
67     core_content_t);
68 #else	/* _LP64 */
69 extern int elfexec(vnode_t *, execa_t *, uarg_t *, intpdata_t *, int,
70     long *, int, caddr_t, cred_t *, int);
71 extern int elfcore(vnode_t *, proc_t *, cred_t *, rlim64_t, int,
72     core_content_t);
73 #endif	/* _LP64 */
74 
75 char _depends_on[] = "exec/elfexec";
76 
77 static struct execsw nesw = {
78 	aout_nmagicstr,
79 	2,
80 	2,
81 	aoutexec,
82 	aoutcore
83 };
84 
85 static struct execsw zesw = {
86 	aout_zmagicstr,
87 	2,
88 	2,
89 	aoutexec,
90 	aoutcore
91 };
92 
93 static struct execsw oesw = {
94 	aout_omagicstr,
95 	2,
96 	2,
97 	aoutexec,
98 	aoutcore
99 };
100 
101 /*
102  * Module linkage information for the kernel.
103  */
104 static struct modlexec nexec = {
105 	&mod_execops, "exec for NMAGIC", &nesw
106 };
107 
108 static struct modlexec zexec = {
109 	&mod_execops, "exec for ZMAGIC", &zesw
110 };
111 
112 static struct modlexec oexec = {
113 	&mod_execops, "exec for OMAGIC", &oesw
114 };
115 
116 static struct modlinkage modlinkage = {
117 	MODREV_1, &nexec, &zexec, &oexec, NULL
118 };
119 
120 int
121 _init(void)
122 {
123 	return (mod_install(&modlinkage));
124 }
125 
126 int
127 _fini(void)
128 {
129 	return (mod_remove(&modlinkage));
130 }
131 
132 int
133 _info(struct modinfo *modinfop)
134 {
135 	return (mod_info(&modlinkage, modinfop));
136 }
137 
138 
139 /*ARGSUSED*/
140 static int
141 aoutexec(vnode_t *vp, struct execa *uap, struct uarg *args,
142     struct intpdata *idatap, int level, long *execsz, int setid,
143     caddr_t exec_file, cred_t *cred, int brand_action)
144 {
145 	int error;
146 	struct exdata edp, edpout;
147 	struct execenv exenv;
148 	proc_t *pp = ttoproc(curthread);
149 	struct vnode *nvp;
150 	int pagetext, pagedata;
151 	int dataprot = PROT_ALL;
152 	int textprot = PROT_ALL & ~PROT_WRITE;
153 	int isdyn;
154 
155 	args->to_model = DATAMODEL_ILP32;
156 	*execsz = btopr(SINCR) + btopr(SSIZE) + btopr(NCARGS32-1);
157 
158 	/*
159 	 * Read in and validate the file header.
160 	 */
161 	if (error = get_aout_head(&vp, &edp, execsz, &isdyn))
162 		return (error);
163 
164 	if (error = chkaout(&edp))
165 		return (error);
166 
167 	/*
168 	 * Take a quick look to see if it looks like we will have
169 	 * enough swap space for the program to get started.  This
170 	 * is not a guarantee that we will succeed, but it is definitely
171 	 * better than finding this out after we are committed to the
172 	 * new memory image.  Maybe what is needed is a way to "prereserve"
173 	 * swap space for some segment mappings here.
174 	 *
175 	 * But with shared libraries the process can make it through
176 	 * the exec only to have ld.so fail to get the program going
177 	 * because its mmap's will not be able to succeed if the system
178 	 * is running low on swap space.  In fact this is a far more
179 	 * common failure mode, but we cannot do much about this here
180 	 * other than add some slop to our anonymous memory resources
181 	 * requirements estimate based on some guess since we cannot know
182 	 * what else the program will really need to get to a useful state.
183 	 *
184 	 * XXX - The stack size (clrnd(SSIZE + btopr(nargc))) should also
185 	 * be used when checking for swap space.  This requires some work
186 	 * since nargc is actually determined in exec_args() which is done
187 	 * after this check and hence we punt for now.
188 	 *
189 	 * nargc = SA(nc + (na + 4) * NBPW) + sizeof (struct rwindow);
190 	 */
191 	if (CURRENT_TOTAL_AVAILABLE_SWAP < btopr(edp.ux_dsize) + btopr(SSIZE))
192 		return (ENOMEM);
193 
194 	/*
195 	 * Load the trap 0 interpreter.
196 	 */
197 	if (error = lookupname("/usr/4lib/sbcp", UIO_SYSSPACE, FOLLOW,
198 	    NULLVPP, &nvp)) {
199 		goto done;
200 	}
201 #ifdef	_LP64
202 	if (error = elf32exec(nvp, uap, args, idatap, level, execsz,
203 	    setid, exec_file, cred, brand_action))
204 #else	/* _LP64 */
205 	if (error = elfexec(nvp, uap, args, idatap, level, execsz,
206 	    setid, exec_file, cred, brand_action))
207 #endif	/* _LP64 */
208 	{
209 		VN_RELE(nvp);
210 		return (error);
211 	}
212 	VN_RELE(nvp);
213 
214 	/*
215 	 * Determine the a.out's characteristics.
216 	 */
217 	getexinfo(&edp, &edpout, &pagetext, &pagedata);
218 
219 	/*
220 	 * Load the a.out's text and data.
221 	 */
222 	if (error = execmap(edp.vp, edp.ux_txtorg, edp.ux_tsize,
223 	    (size_t)0, edp.ux_toffset, textprot, pagetext, 0))
224 		goto done;
225 	if (error = execmap(edp.vp, edp.ux_datorg, edp.ux_dsize,
226 	    edp.ux_bsize, edp.ux_doffset, dataprot, pagedata, 0))
227 		goto done;
228 
229 	exenv.ex_bssbase = (caddr_t)edp.ux_datorg;
230 	exenv.ex_brkbase = (caddr_t)edp.ux_datorg;
231 	exenv.ex_brksize = edp.ux_dsize + edp.ux_bsize;
232 	exenv.ex_magic = edp.ux_mag;
233 	exenv.ex_vp = edp.vp;
234 	setexecenv(&exenv);
235 
236 done:
237 	if (error != 0)
238 		psignal(pp, SIGKILL);
239 	else {
240 		/*
241 		 * Ensure that the max fds do not exceed 256 (this is
242 		 * applicable to 4.x binaries, which is why we only
243 		 * do it on a.out files).
244 		 */
245 		struct rlimit64 fdno_rlim;
246 		rctl_alloc_gp_t *gp = rctl_rlimit_set_prealloc(1);
247 
248 		mutex_enter(&curproc->p_lock);
249 		(void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_NOFILE], curproc,
250 		    &fdno_rlim);
251 		if (fdno_rlim.rlim_cur > 256) {
252 			fdno_rlim.rlim_cur = fdno_rlim.rlim_max = 256;
253 			(void) rctl_rlimit_set(rctlproc_legacy[RLIMIT_NOFILE],
254 			    curproc, &fdno_rlim, gp,
255 			    rctlproc_flags[RLIMIT_NOFILE],
256 			    rctlproc_signals[RLIMIT_NOFILE], CRED());
257 		} else if (fdno_rlim.rlim_max > 256) {
258 			fdno_rlim.rlim_max = 256;
259 			(void) rctl_rlimit_set(rctlproc_legacy[RLIMIT_NOFILE],
260 			    curproc, &fdno_rlim, gp,
261 			    rctlproc_flags[RLIMIT_NOFILE],
262 			    rctlproc_signals[RLIMIT_NOFILE], CRED());
263 		}
264 		mutex_exit(&curproc->p_lock);
265 
266 		rctl_prealloc_destroy(gp);
267 	}
268 
269 	return (error);
270 }
271 
272 /*
273  * Read in and validate the file header.
274  */
275 static int
276 get_aout_head(struct vnode **vpp, struct exdata *edp, long *execsz, int *isdyn)
277 {
278 	struct vnode *vp = *vpp;
279 	struct exec filhdr;
280 	int error;
281 	ssize_t resid;
282 	rlim64_t limit;
283 	rlim64_t roundlimit;
284 
285 	if (error = vn_rdwr(UIO_READ, vp, (caddr_t)&filhdr,
286 	    (ssize_t)sizeof (filhdr), (offset_t)0, UIO_SYSSPACE, 0,
287 	    (rlim64_t)0, CRED(), &resid))
288 		return (error);
289 
290 	if (resid != 0)
291 		return (ENOEXEC);
292 
293 	switch (filhdr.a_magic) {
294 	case OMAGIC:
295 		filhdr.a_data += filhdr.a_text;
296 		filhdr.a_text = 0;
297 		break;
298 	case ZMAGIC:
299 	case NMAGIC:
300 		break;
301 	default:
302 		return (ENOEXEC);
303 	}
304 
305 	/*
306 	 * Check total memory requirements (in pages) for a new process
307 	 * against the available memory or upper limit of memory allowed.
308 	 *
309 	 * For the 64-bit kernel, the limit can be set large enough so that
310 	 * rounding it up to a page can overflow, so we check for btopr()
311 	 * overflowing here by comparing it with the unrounded limit in pages.
312 	 */
313 	*execsz += btopr(filhdr.a_text + filhdr.a_data);
314 	limit = btop(curproc->p_vmem_ctl);
315 	roundlimit = btopr(curproc->p_vmem_ctl);
316 	if ((roundlimit > limit && *execsz > roundlimit) ||
317 	    (roundlimit < limit && *execsz > limit)) {
318 		mutex_enter(&curproc->p_lock);
319 		(void) rctl_action(rctlproc_legacy[RLIMIT_VMEM],
320 		    curproc->p_rctls, curproc, RCA_SAFE);
321 		mutex_exit(&curproc->p_lock);
322 		return (ENOMEM);
323 	}
324 
325 	edp->ux_mach = filhdr.a_machtype;
326 	edp->ux_tsize = filhdr.a_text;
327 	edp->ux_dsize = filhdr.a_data;
328 	edp->ux_bsize = filhdr.a_bss;
329 	edp->ux_mag = filhdr.a_magic;
330 	edp->ux_toffset = gettfile(&filhdr);
331 	edp->ux_doffset = getdfile(&filhdr);
332 	edp->ux_txtorg = gettmem(&filhdr);
333 	edp->ux_datorg = getdmem(&filhdr);
334 	edp->ux_entloc = (caddr_t)(uintptr_t)filhdr.a_entry;
335 	edp->vp = vp;
336 	*isdyn = filhdr.a_dynamic;
337 
338 	return (0);
339 }
340 
341 static int
342 aoutcore(vnode_t *vp, proc_t *pp, struct cred *credp, rlim64_t rlimit, int sig,
343     core_content_t content)
344 {
345 #ifdef	_LP64
346 	return (elf32core(vp, pp, credp, rlimit, sig, content));
347 #else
348 	return (elfcore(vp, pp, credp, rlimit, sig, content));
349 #endif
350 }
351