xref: /freebsd/sys/amd64/linux/linux_sysvec.c (revision 3332f1b444d4a73238e9f59cca27bfc95fe936bd)
1 /*-
2  * Copyright (c) 2004 Tim J. Robbins
3  * Copyright (c) 2003 Peter Wemm
4  * Copyright (c) 2002 Doug Rabson
5  * Copyright (c) 1998-1999 Andrew Gallatin
6  * Copyright (c) 1994-1996 Søren Schmidt
7  * All rights reserved.
8  * Copyright (c) 2013, 2021 Dmitry Chagin <dchagin@FreeBSD.org>
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer
15  *    in this position and unchanged.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  * 3. The name of the author may not be used to endorse or promote products
20  *    derived from this software without specific prior written permission
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
27  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
31  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include <sys/cdefs.h>
35 __FBSDID("$FreeBSD$");
36 
37 #define	__ELF_WORD_SIZE	64
38 
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/exec.h>
42 #include <sys/fcntl.h>
43 #include <sys/imgact.h>
44 #include <sys/imgact_elf.h>
45 #include <sys/kernel.h>
46 #include <sys/ktr.h>
47 #include <sys/lock.h>
48 #include <sys/malloc.h>
49 #include <sys/module.h>
50 #include <sys/mutex.h>
51 #include <sys/proc.h>
52 #include <sys/resourcevar.h>
53 #include <sys/stddef.h>
54 #include <sys/signalvar.h>
55 #include <sys/syscallsubr.h>
56 #include <sys/sysctl.h>
57 #include <sys/sysent.h>
58 #include <sys/sysproto.h>
59 #include <sys/vnode.h>
60 #include <sys/eventhandler.h>
61 
62 #include <vm/vm.h>
63 #include <vm/pmap.h>
64 #include <vm/vm_extern.h>
65 #include <vm/vm_map.h>
66 #include <vm/vm_object.h>
67 #include <vm/vm_page.h>
68 #include <vm/vm_param.h>
69 
70 #include <machine/cpu.h>
71 #include <machine/md_var.h>
72 #include <machine/pcb.h>
73 #include <machine/specialreg.h>
74 #include <machine/trap.h>
75 
76 #include <x86/linux/linux_x86.h>
77 #include <amd64/linux/linux.h>
78 #include <amd64/linux/linux_proto.h>
79 #include <compat/linux/linux_emul.h>
80 #include <compat/linux/linux_fork.h>
81 #include <compat/linux/linux_ioctl.h>
82 #include <compat/linux/linux_mib.h>
83 #include <compat/linux/linux_misc.h>
84 #include <compat/linux/linux_signal.h>
85 #include <compat/linux/linux_sysproto.h>
86 #include <compat/linux/linux_util.h>
87 #include <compat/linux/linux_vdso.h>
88 
89 MODULE_VERSION(linux64, 1);
90 
91 #define	LINUX_VDSOPAGE_SIZE	PAGE_SIZE * 2
92 #define	LINUX_VDSOPAGE_LA48	(VM_MAXUSER_ADDRESS_LA48 - \
93 				    LINUX_VDSOPAGE_SIZE)
94 #define	LINUX_SHAREDPAGE_LA48	(LINUX_VDSOPAGE_LA48 - PAGE_SIZE)
95 				/*
96 				 * PAGE_SIZE - the size
97 				 * of the native SHAREDPAGE
98 				 */
99 #define	LINUX_USRSTACK_LA48	LINUX_SHAREDPAGE_LA48
100 #define	LINUX_PS_STRINGS_LA48	(LINUX_USRSTACK_LA48 - \
101 				    sizeof(struct ps_strings))
102 
103 static int linux_szsigcode;
104 static vm_object_t linux_vdso_obj;
105 static char *linux_vdso_mapping;
106 extern char _binary_linux_vdso_so_o_start;
107 extern char _binary_linux_vdso_so_o_end;
108 static vm_offset_t linux_vdso_base;
109 
110 extern struct sysent linux_sysent[LINUX_SYS_MAXSYSCALL];
111 
112 SET_DECLARE(linux_ioctl_handler_set, struct linux_ioctl_handler);
113 
114 static int	linux_copyout_strings(struct image_params *imgp,
115 		    uintptr_t *stack_base);
116 static int	linux_fixup_elf(uintptr_t *stack_base,
117 		    struct image_params *iparams);
118 static bool	linux_trans_osrel(const Elf_Note *note, int32_t *osrel);
119 static void	linux_vdso_install(const void *param);
120 static void	linux_vdso_deinstall(const void *param);
121 static void	linux_vdso_reloc(char *mapping, Elf_Addr offset);
122 static void	linux_set_syscall_retval(struct thread *td, int error);
123 static int	linux_fetch_syscall_args(struct thread *td);
124 static void	linux_exec_setregs(struct thread *td, struct image_params *imgp,
125 		    uintptr_t stack);
126 static void	linux_exec_sysvec_init(void *param);
127 static int	linux_on_exec_vmspace(struct proc *p,
128 		    struct image_params *imgp);
129 static void	linux_set_fork_retval(struct thread *td);
130 static int	linux_vsyscall(struct thread *td);
131 
132 #define LINUX_T_UNKNOWN  255
133 static int _bsd_to_linux_trapcode[] = {
134 	LINUX_T_UNKNOWN,	/* 0 */
135 	6,			/* 1  T_PRIVINFLT */
136 	LINUX_T_UNKNOWN,	/* 2 */
137 	3,			/* 3  T_BPTFLT */
138 	LINUX_T_UNKNOWN,	/* 4 */
139 	LINUX_T_UNKNOWN,	/* 5 */
140 	16,			/* 6  T_ARITHTRAP */
141 	254,			/* 7  T_ASTFLT */
142 	LINUX_T_UNKNOWN,	/* 8 */
143 	13,			/* 9  T_PROTFLT */
144 	1,			/* 10 T_TRCTRAP */
145 	LINUX_T_UNKNOWN,	/* 11 */
146 	14,			/* 12 T_PAGEFLT */
147 	LINUX_T_UNKNOWN,	/* 13 */
148 	17,			/* 14 T_ALIGNFLT */
149 	LINUX_T_UNKNOWN,	/* 15 */
150 	LINUX_T_UNKNOWN,	/* 16 */
151 	LINUX_T_UNKNOWN,	/* 17 */
152 	0,			/* 18 T_DIVIDE */
153 	2,			/* 19 T_NMI */
154 	4,			/* 20 T_OFLOW */
155 	5,			/* 21 T_BOUND */
156 	7,			/* 22 T_DNA */
157 	8,			/* 23 T_DOUBLEFLT */
158 	9,			/* 24 T_FPOPFLT */
159 	10,			/* 25 T_TSSFLT */
160 	11,			/* 26 T_SEGNPFLT */
161 	12,			/* 27 T_STKFLT */
162 	18,			/* 28 T_MCHK */
163 	19,			/* 29 T_XMMFLT */
164 	15			/* 30 T_RESERVED */
165 };
166 #define bsd_to_linux_trapcode(code) \
167     ((code)<nitems(_bsd_to_linux_trapcode)? \
168      _bsd_to_linux_trapcode[(code)]: \
169      LINUX_T_UNKNOWN)
170 
171 LINUX_VDSO_SYM_INTPTR(linux_rt_sigcode);
172 LINUX_VDSO_SYM_CHAR(linux_platform);
173 LINUX_VDSO_SYM_INTPTR(kern_timekeep_base);
174 LINUX_VDSO_SYM_INTPTR(kern_tsc_selector);
175 
176 /*
177  * If FreeBSD & Linux have a difference of opinion about what a trap
178  * means, deal with it here.
179  *
180  * MPSAFE
181  */
182 static int
183 linux_translate_traps(int signal, int trap_code)
184 {
185 
186 	if (signal != SIGBUS)
187 		return (signal);
188 	switch (trap_code) {
189 	case T_PROTFLT:
190 	case T_TSSFLT:
191 	case T_DOUBLEFLT:
192 	case T_PAGEFLT:
193 		return (SIGSEGV);
194 	default:
195 		return (signal);
196 	}
197 }
198 
199 static int
200 linux_fetch_syscall_args(struct thread *td)
201 {
202 	struct proc *p;
203 	struct trapframe *frame;
204 	struct syscall_args *sa;
205 
206 	p = td->td_proc;
207 	frame = td->td_frame;
208 	sa = &td->td_sa;
209 
210 	sa->args[0] = frame->tf_rdi;
211 	sa->args[1] = frame->tf_rsi;
212 	sa->args[2] = frame->tf_rdx;
213 	sa->args[3] = frame->tf_rcx;
214 	sa->args[4] = frame->tf_r8;
215 	sa->args[5] = frame->tf_r9;
216 	sa->code = frame->tf_rax;
217 	sa->original_code = sa->code;
218 
219 	if (sa->code >= p->p_sysent->sv_size)
220 		/* nosys */
221 		sa->callp = &p->p_sysent->sv_table[p->p_sysent->sv_size - 1];
222 	else
223 		sa->callp = &p->p_sysent->sv_table[sa->code];
224 
225 	td->td_retval[0] = 0;
226 	return (0);
227 }
228 
229 static void
230 linux_set_syscall_retval(struct thread *td, int error)
231 {
232 	struct trapframe *frame;
233 
234 	frame = td->td_frame;
235 
236 	switch (error) {
237 	case 0:
238 		frame->tf_rax = td->td_retval[0];
239 		frame->tf_r10 = frame->tf_rcx;
240 		break;
241 
242 	case ERESTART:
243 		/*
244 		 * Reconstruct pc, we know that 'syscall' is 2 bytes,
245 		 * lcall $X,y is 7 bytes, int 0x80 is 2 bytes.
246 		 * We saved this in tf_err.
247 		 *
248 		 */
249 		frame->tf_rip -= frame->tf_err;
250 		frame->tf_r10 = frame->tf_rcx;
251 		break;
252 
253 	case EJUSTRETURN:
254 		break;
255 
256 	default:
257 		frame->tf_rax = bsd_to_linux_errno(error);
258 		frame->tf_r10 = frame->tf_rcx;
259 		break;
260 	}
261 
262 	/*
263 	 * Differently from FreeBSD native ABI, on Linux only %rcx
264 	 * and %r11 values are not preserved across the syscall.
265 	 * Require full context restore to get all registers except
266 	 * those two restored at return to usermode.
267 	 *
268 	 * XXX: Would be great to be able to avoid PCB_FULL_IRET
269 	 *      for the error == 0 case.
270 	 */
271 	set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
272 }
273 
274 static void
275 linux_set_fork_retval(struct thread *td)
276 {
277 	struct trapframe *frame = td->td_frame;
278 
279 	frame->tf_rax = 0;
280 }
281 
282 static int
283 linux_copyout_auxargs(struct image_params *imgp, uintptr_t base)
284 {
285 	Elf_Auxargs *args;
286 	Elf_Auxinfo *argarray, *pos;
287 	struct proc *p;
288 	int error, issetugid;
289 
290 	p = imgp->proc;
291 	args = (Elf64_Auxargs *)imgp->auxargs;
292 	argarray = pos = malloc(LINUX_AT_COUNT * sizeof(*pos), M_TEMP,
293 	    M_WAITOK | M_ZERO);
294 
295 	issetugid = p->p_flag & P_SUGID ? 1 : 0;
296 	AUXARGS_ENTRY(pos, LINUX_AT_SYSINFO_EHDR, linux_vdso_base);
297 	AUXARGS_ENTRY(pos, LINUX_AT_HWCAP, cpu_feature);
298 	AUXARGS_ENTRY(pos, AT_PAGESZ, args->pagesz);
299 	AUXARGS_ENTRY(pos, LINUX_AT_CLKTCK, stclohz);
300 	AUXARGS_ENTRY(pos, AT_PHDR, args->phdr);
301 	AUXARGS_ENTRY(pos, AT_PHENT, args->phent);
302 	AUXARGS_ENTRY(pos, AT_PHNUM, args->phnum);
303 	AUXARGS_ENTRY(pos, AT_BASE, args->base);
304 	AUXARGS_ENTRY(pos, AT_FLAGS, args->flags);
305 	AUXARGS_ENTRY(pos, AT_ENTRY, args->entry);
306 	AUXARGS_ENTRY(pos, AT_UID, imgp->proc->p_ucred->cr_ruid);
307 	AUXARGS_ENTRY(pos, AT_EUID, imgp->proc->p_ucred->cr_svuid);
308 	AUXARGS_ENTRY(pos, AT_GID, imgp->proc->p_ucred->cr_rgid);
309 	AUXARGS_ENTRY(pos, AT_EGID, imgp->proc->p_ucred->cr_svgid);
310 	AUXARGS_ENTRY(pos, LINUX_AT_SECURE, issetugid);
311 	AUXARGS_ENTRY_PTR(pos, LINUX_AT_RANDOM, imgp->canary);
312 	AUXARGS_ENTRY(pos, LINUX_AT_HWCAP2, 0);
313 	if (imgp->execpathp != 0)
314 		AUXARGS_ENTRY_PTR(pos, LINUX_AT_EXECFN, imgp->execpathp);
315 	if (args->execfd != -1)
316 		AUXARGS_ENTRY(pos, AT_EXECFD, args->execfd);
317 	AUXARGS_ENTRY(pos, LINUX_AT_PLATFORM, PTROUT(linux_platform));
318 	AUXARGS_ENTRY(pos, AT_NULL, 0);
319 
320 	free(imgp->auxargs, M_TEMP);
321 	imgp->auxargs = NULL;
322 	KASSERT(pos - argarray <= LINUX_AT_COUNT, ("Too many auxargs"));
323 
324 	error = copyout(argarray, (void *)base,
325 	    sizeof(*argarray) * LINUX_AT_COUNT);
326 	free(argarray, M_TEMP);
327 	return (error);
328 }
329 
330 static int
331 linux_fixup_elf(uintptr_t *stack_base, struct image_params *imgp)
332 {
333 	Elf_Addr *base;
334 
335 	base = (Elf64_Addr *)*stack_base;
336 	base--;
337 	if (suword(base, (uint64_t)imgp->args->argc) == -1)
338 		return (EFAULT);
339 
340 	*stack_base = (uintptr_t)base;
341 	return (0);
342 }
343 
344 /*
345  * Copy strings out to the new process address space, constructing new arg
346  * and env vector tables. Return a pointer to the base so that it can be used
347  * as the initial stack pointer.
348  */
349 static int
350 linux_copyout_strings(struct image_params *imgp, uintptr_t *stack_base)
351 {
352 	int argc, envc, error;
353 	char **vectp;
354 	char *stringp;
355 	uintptr_t destp, ustringp;
356 	struct ps_strings *arginfo;
357 	char canary[LINUX_AT_RANDOM_LEN];
358 	size_t execpath_len;
359 	struct proc *p;
360 
361 	/* Calculate string base and vector table pointers. */
362 	if (imgp->execpath != NULL && imgp->auxargs != NULL)
363 		execpath_len = strlen(imgp->execpath) + 1;
364 	else
365 		execpath_len = 0;
366 
367 	p = imgp->proc;
368 	arginfo = (struct ps_strings *)p->p_sysent->sv_psstrings;
369 	destp = (uintptr_t)arginfo;
370 
371 	if (execpath_len != 0) {
372 		destp -= execpath_len;
373 		destp = rounddown2(destp, sizeof(void *));
374 		imgp->execpathp = (void *)destp;
375 		error = copyout(imgp->execpath, imgp->execpathp, execpath_len);
376 		if (error != 0)
377 			return (error);
378 	}
379 
380 	/* Prepare the canary for SSP. */
381 	arc4rand(canary, sizeof(canary), 0);
382 	destp -= roundup(sizeof(canary), sizeof(void *));
383 	imgp->canary = (void *)destp;
384 	error = copyout(canary, imgp->canary, sizeof(canary));
385 	if (error != 0)
386 		return (error);
387 
388 	/* Allocate room for the argument and environment strings. */
389 	destp -= ARG_MAX - imgp->args->stringspace;
390 	destp = rounddown2(destp, sizeof(void *));
391 	ustringp = destp;
392 
393 	if (imgp->auxargs) {
394 		/*
395 		 * Allocate room on the stack for the ELF auxargs
396 		 * array.  It has LINUX_AT_COUNT entries.
397 		 */
398 		destp -= LINUX_AT_COUNT * sizeof(Elf64_Auxinfo);
399 		destp = rounddown2(destp, sizeof(void *));
400 	}
401 
402 	vectp = (char **)destp;
403 
404 	/*
405 	 * Allocate room for the argv[] and env vectors including the
406 	 * terminating NULL pointers.
407 	 */
408 	vectp -= imgp->args->argc + 1 + imgp->args->envc + 1;
409 
410 	/*
411 	 * Starting with 2.24, glibc depends on a 16-byte stack alignment.
412 	 * One "long argc" will be prepended later.
413 	 */
414 	vectp = (char **)((((uintptr_t)vectp + 8) & ~0xF) - 8);
415 
416 	/* vectp also becomes our initial stack base. */
417 	*stack_base = (uintptr_t)vectp;
418 
419 	stringp = imgp->args->begin_argv;
420 	argc = imgp->args->argc;
421 	envc = imgp->args->envc;
422 
423 	/* Copy out strings - arguments and environment. */
424 	error = copyout(stringp, (void *)ustringp,
425 	    ARG_MAX - imgp->args->stringspace);
426 	if (error != 0)
427 		return (error);
428 
429 	/* Fill in "ps_strings" struct for ps, w, etc. */
430 	if (suword(&arginfo->ps_argvstr, (long)(intptr_t)vectp) != 0 ||
431 	    suword(&arginfo->ps_nargvstr, argc) != 0)
432 		return (EFAULT);
433 
434 	/* Fill in argument portion of vector table. */
435 	for (; argc > 0; --argc) {
436 		if (suword(vectp++, ustringp) != 0)
437 			return (EFAULT);
438 		while (*stringp++ != 0)
439 			ustringp++;
440 		ustringp++;
441 	}
442 
443 	/* A null vector table pointer separates the argp's from the envp's. */
444 	if (suword(vectp++, 0) != 0)
445 		return (EFAULT);
446 
447 	if (suword(&arginfo->ps_envstr, (long)(intptr_t)vectp) != 0 ||
448 	    suword(&arginfo->ps_nenvstr, envc) != 0)
449 		return (EFAULT);
450 
451 	/* Fill in environment portion of vector table. */
452 	for (; envc > 0; --envc) {
453 		if (suword(vectp++, ustringp) != 0)
454 			return (EFAULT);
455 		while (*stringp++ != 0)
456 			ustringp++;
457 		ustringp++;
458 	}
459 
460 	/* The end of the vector table is a null pointer. */
461 	if (suword(vectp, 0) != 0)
462 		return (EFAULT);
463 
464 	if (imgp->auxargs) {
465 		vectp++;
466 		error = imgp->sysent->sv_copyout_auxargs(imgp,
467 		    (uintptr_t)vectp);
468 		if (error != 0)
469 			return (error);
470 	}
471 
472 	return (0);
473 }
474 
475 /*
476  * Reset registers to default values on exec.
477  */
478 static void
479 linux_exec_setregs(struct thread *td, struct image_params *imgp,
480     uintptr_t stack)
481 {
482 	struct trapframe *regs;
483 	struct pcb *pcb;
484 	register_t saved_rflags;
485 
486 	regs = td->td_frame;
487 	pcb = td->td_pcb;
488 
489 	if (td->td_proc->p_md.md_ldt != NULL)
490 		user_ldt_free(td);
491 
492 	pcb->pcb_fsbase = 0;
493 	pcb->pcb_gsbase = 0;
494 	clear_pcb_flags(pcb, PCB_32BIT);
495 	pcb->pcb_initial_fpucw = __LINUX_NPXCW__;
496 	set_pcb_flags(pcb, PCB_FULL_IRET);
497 
498 	saved_rflags = regs->tf_rflags & PSL_T;
499 	bzero((char *)regs, sizeof(struct trapframe));
500 	regs->tf_rip = imgp->entry_addr;
501 	regs->tf_rsp = stack;
502 	regs->tf_rflags = PSL_USER | saved_rflags;
503 	regs->tf_ss = _udatasel;
504 	regs->tf_cs = _ucodesel;
505 	regs->tf_ds = _udatasel;
506 	regs->tf_es = _udatasel;
507 	regs->tf_fs = _ufssel;
508 	regs->tf_gs = _ugssel;
509 	regs->tf_flags = TF_HASSEGS;
510 
511 	x86_clear_dbregs(pcb);
512 
513 	/*
514 	 * Drop the FP state if we hold it, so that the process gets a
515 	 * clean FP state if it uses the FPU again.
516 	 */
517 	fpstate_drop(td);
518 }
519 
520 /*
521  * Copied from amd64/amd64/machdep.c
522  *
523  * XXX fpu state need? don't think so
524  */
525 int
526 linux_rt_sigreturn(struct thread *td, struct linux_rt_sigreturn_args *args)
527 {
528 	struct proc *p;
529 	struct l_ucontext uc;
530 	struct l_sigcontext *context;
531 	struct trapframe *regs;
532 	unsigned long rflags;
533 	int error;
534 	ksiginfo_t ksi;
535 
536 	regs = td->td_frame;
537 	error = copyin((void *)regs->tf_rbx, &uc, sizeof(uc));
538 	if (error != 0)
539 		return (error);
540 
541 	p = td->td_proc;
542 	context = &uc.uc_mcontext;
543 	rflags = context->sc_rflags;
544 
545 	/*
546 	 * Don't allow users to change privileged or reserved flags.
547 	 */
548 	/*
549 	 * XXX do allow users to change the privileged flag PSL_RF.
550 	 * The cpu sets PSL_RF in tf_rflags for faults.  Debuggers
551 	 * should sometimes set it there too.  tf_rflags is kept in
552 	 * the signal context during signal handling and there is no
553 	 * other place to remember it, so the PSL_RF bit may be
554 	 * corrupted by the signal handler without us knowing.
555 	 * Corruption of the PSL_RF bit at worst causes one more or
556 	 * one less debugger trap, so allowing it is fairly harmless.
557 	 */
558 	if (!EFL_SECURE(rflags & ~PSL_RF, regs->tf_rflags & ~PSL_RF)) {
559 		uprintf("pid %d comm %s linux mangled rflags %#lx\n",
560 		    p->p_pid, p->p_comm, rflags);
561 		return (EINVAL);
562 	}
563 
564 	/*
565 	 * Don't allow users to load a valid privileged %cs.  Let the
566 	 * hardware check for invalid selectors, excess privilege in
567 	 * other selectors, invalid %eip's and invalid %esp's.
568 	 */
569 	if (!CS_SECURE(context->sc_cs)) {
570 		uprintf("pid %d comm %s linux mangled cs %#x\n",
571 		    p->p_pid, p->p_comm, context->sc_cs);
572 		ksiginfo_init_trap(&ksi);
573 		ksi.ksi_signo = SIGBUS;
574 		ksi.ksi_code = BUS_OBJERR;
575 		ksi.ksi_trapno = T_PROTFLT;
576 		ksi.ksi_addr = (void *)regs->tf_rip;
577 		trapsignal(td, &ksi);
578 		return (EINVAL);
579 	}
580 
581 	PROC_LOCK(p);
582 	linux_to_bsd_sigset(&uc.uc_sigmask, &td->td_sigmask);
583 	SIG_CANTMASK(td->td_sigmask);
584 	signotify(td);
585 	PROC_UNLOCK(p);
586 
587 	regs->tf_rdi    = context->sc_rdi;
588 	regs->tf_rsi    = context->sc_rsi;
589 	regs->tf_rdx    = context->sc_rdx;
590 	regs->tf_rbp    = context->sc_rbp;
591 	regs->tf_rbx    = context->sc_rbx;
592 	regs->tf_rcx    = context->sc_rcx;
593 	regs->tf_rax    = context->sc_rax;
594 	regs->tf_rip    = context->sc_rip;
595 	regs->tf_rsp    = context->sc_rsp;
596 	regs->tf_r8     = context->sc_r8;
597 	regs->tf_r9     = context->sc_r9;
598 	regs->tf_r10    = context->sc_r10;
599 	regs->tf_r11    = context->sc_r11;
600 	regs->tf_r12    = context->sc_r12;
601 	regs->tf_r13    = context->sc_r13;
602 	regs->tf_r14    = context->sc_r14;
603 	regs->tf_r15    = context->sc_r15;
604 	regs->tf_cs     = context->sc_cs;
605 	regs->tf_err    = context->sc_err;
606 	regs->tf_rflags = rflags;
607 
608 	set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
609 	return (EJUSTRETURN);
610 }
611 
612 /*
613  * copied from amd64/amd64/machdep.c
614  *
615  * Send an interrupt to process.
616  */
617 static void
618 linux_rt_sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask)
619 {
620 	struct l_rt_sigframe sf, *sfp;
621 	struct proc *p;
622 	struct thread *td;
623 	struct sigacts *psp;
624 	caddr_t sp;
625 	struct trapframe *regs;
626 	int sig, code;
627 	int oonstack;
628 
629 	td = curthread;
630 	p = td->td_proc;
631 	PROC_LOCK_ASSERT(p, MA_OWNED);
632 	sig = ksi->ksi_signo;
633 	psp = p->p_sigacts;
634 	code = ksi->ksi_code;
635 	mtx_assert(&psp->ps_mtx, MA_OWNED);
636 	regs = td->td_frame;
637 	oonstack = sigonstack(regs->tf_rsp);
638 
639 	LINUX_CTR4(rt_sendsig, "%p, %d, %p, %u",
640 	    catcher, sig, mask, code);
641 
642 	/* Save user context. */
643 	bzero(&sf, sizeof(sf));
644 	bsd_to_linux_sigset(mask, &sf.sf_sc.uc_sigmask);
645 	bsd_to_linux_sigset(mask, &sf.sf_sc.uc_mcontext.sc_mask);
646 
647 	sf.sf_sc.uc_stack.ss_sp = PTROUT(td->td_sigstk.ss_sp);
648 	sf.sf_sc.uc_stack.ss_size = td->td_sigstk.ss_size;
649 	sf.sf_sc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK)
650 	    ? ((oonstack) ? LINUX_SS_ONSTACK : 0) : LINUX_SS_DISABLE;
651 
652 	sf.sf_sc.uc_mcontext.sc_rdi    = regs->tf_rdi;
653 	sf.sf_sc.uc_mcontext.sc_rsi    = regs->tf_rsi;
654 	sf.sf_sc.uc_mcontext.sc_rdx    = regs->tf_rdx;
655 	sf.sf_sc.uc_mcontext.sc_rbp    = regs->tf_rbp;
656 	sf.sf_sc.uc_mcontext.sc_rbx    = regs->tf_rbx;
657 	sf.sf_sc.uc_mcontext.sc_rcx    = regs->tf_rcx;
658 	sf.sf_sc.uc_mcontext.sc_rax    = regs->tf_rax;
659 	sf.sf_sc.uc_mcontext.sc_rip    = regs->tf_rip;
660 	sf.sf_sc.uc_mcontext.sc_rsp    = regs->tf_rsp;
661 	sf.sf_sc.uc_mcontext.sc_r8     = regs->tf_r8;
662 	sf.sf_sc.uc_mcontext.sc_r9     = regs->tf_r9;
663 	sf.sf_sc.uc_mcontext.sc_r10    = regs->tf_r10;
664 	sf.sf_sc.uc_mcontext.sc_r11    = regs->tf_r11;
665 	sf.sf_sc.uc_mcontext.sc_r12    = regs->tf_r12;
666 	sf.sf_sc.uc_mcontext.sc_r13    = regs->tf_r13;
667 	sf.sf_sc.uc_mcontext.sc_r14    = regs->tf_r14;
668 	sf.sf_sc.uc_mcontext.sc_r15    = regs->tf_r15;
669 	sf.sf_sc.uc_mcontext.sc_cs     = regs->tf_cs;
670 	sf.sf_sc.uc_mcontext.sc_rflags = regs->tf_rflags;
671 	sf.sf_sc.uc_mcontext.sc_err    = regs->tf_err;
672 	sf.sf_sc.uc_mcontext.sc_trapno = bsd_to_linux_trapcode(code);
673 	sf.sf_sc.uc_mcontext.sc_cr2    = (register_t)ksi->ksi_addr;
674 
675 	/* Allocate space for the signal handler context. */
676 	if ((td->td_pflags & TDP_ALTSTACK) != 0 && !oonstack &&
677 	    SIGISMEMBER(psp->ps_sigonstack, sig)) {
678 		sp = (caddr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size -
679 		    sizeof(struct l_rt_sigframe);
680 	} else
681 		sp = (caddr_t)regs->tf_rsp - sizeof(struct l_rt_sigframe) - 128;
682 	/* Align to 16 bytes. */
683 	sfp = (struct l_rt_sigframe *)((unsigned long)sp & ~0xFul);
684 
685 	/* Translate the signal. */
686 	sig = bsd_to_linux_signal(sig);
687 
688 	/* Build the argument list for the signal handler. */
689 	regs->tf_rdi = sig;			/* arg 1 in %rdi */
690 	regs->tf_rax = 0;
691 	regs->tf_rsi = (register_t)&sfp->sf_si;	/* arg 2 in %rsi */
692 	regs->tf_rdx = (register_t)&sfp->sf_sc;	/* arg 3 in %rdx */
693 
694 	/* Fill in POSIX parts. */
695 	siginfo_to_lsiginfo(&ksi->ksi_info, &sf.sf_si, sig);
696 	sf.sf_handler = catcher;
697 
698 	mtx_unlock(&psp->ps_mtx);
699 	PROC_UNLOCK(p);
700 
701 	/* Copy the sigframe out to the user's stack. */
702 	if (copyout(&sf, sfp, sizeof(*sfp)) != 0) {
703 		PROC_LOCK(p);
704 		sigexit(td, SIGILL);
705 	}
706 
707 	regs->tf_rsp = (long)sfp;
708 	regs->tf_rip = linux_rt_sigcode;
709 	regs->tf_rflags &= ~(PSL_T | PSL_D);
710 	regs->tf_cs = _ucodesel;
711 	set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
712 	PROC_LOCK(p);
713 	mtx_lock(&psp->ps_mtx);
714 }
715 
716 #define	LINUX_VSYSCALL_START		(-10UL << 20)
717 #define	LINUX_VSYSCALL_SZ		1024
718 
719 const unsigned long linux_vsyscall_vector[] = {
720 	LINUX_SYS_gettimeofday,
721 	LINUX_SYS_linux_time,
722 	LINUX_SYS_linux_getcpu,
723 };
724 
725 static int
726 linux_vsyscall(struct thread *td)
727 {
728 	struct trapframe *frame;
729 	uint64_t retqaddr;
730 	int code, traced;
731 	int error;
732 
733 	frame = td->td_frame;
734 
735 	/* Check %rip for vsyscall area. */
736 	if (__predict_true(frame->tf_rip < LINUX_VSYSCALL_START))
737 		return (EINVAL);
738 	if ((frame->tf_rip & (LINUX_VSYSCALL_SZ - 1)) != 0)
739 		return (EINVAL);
740 	code = (frame->tf_rip - LINUX_VSYSCALL_START) / LINUX_VSYSCALL_SZ;
741 	if (code >= nitems(linux_vsyscall_vector))
742 		return (EINVAL);
743 
744 	/*
745 	 * vsyscall called as callq *(%rax), so we must
746 	 * use return address from %rsp and also fixup %rsp.
747 	 */
748 	error = copyin((void *)frame->tf_rsp, &retqaddr, sizeof(retqaddr));
749 	if (error)
750 		return (error);
751 
752 	frame->tf_rip = retqaddr;
753 	frame->tf_rax = linux_vsyscall_vector[code];
754 	frame->tf_rsp += 8;
755 
756 	traced = (frame->tf_flags & PSL_T);
757 
758 	amd64_syscall(td, traced);
759 
760 	return (0);
761 }
762 
763 struct sysentvec elf_linux_sysvec = {
764 	.sv_size	= LINUX_SYS_MAXSYSCALL,
765 	.sv_table	= linux_sysent,
766 	.sv_transtrap	= linux_translate_traps,
767 	.sv_fixup	= linux_fixup_elf,
768 	.sv_sendsig	= linux_rt_sendsig,
769 	.sv_sigcode	= &_binary_linux_vdso_so_o_start,
770 	.sv_szsigcode	= &linux_szsigcode,
771 	.sv_name	= "Linux ELF64",
772 	.sv_coredump	= elf64_coredump,
773 	.sv_elf_core_osabi = ELFOSABI_NONE,
774 	.sv_elf_core_abi_vendor = LINUX_ABI_VENDOR,
775 	.sv_elf_core_prepare_notes = linux64_prepare_notes,
776 	.sv_imgact_try	= linux_exec_imgact_try,
777 	.sv_minsigstksz	= LINUX_MINSIGSTKSZ,
778 	.sv_minuser	= VM_MIN_ADDRESS,
779 	.sv_maxuser	= VM_MAXUSER_ADDRESS_LA48,
780 	.sv_usrstack	= LINUX_USRSTACK_LA48,
781 	.sv_psstrings	= LINUX_PS_STRINGS_LA48,
782 	.sv_stackprot	= VM_PROT_ALL,
783 	.sv_copyout_auxargs = linux_copyout_auxargs,
784 	.sv_copyout_strings = linux_copyout_strings,
785 	.sv_setregs	= linux_exec_setregs,
786 	.sv_fixlimit	= NULL,
787 	.sv_maxssiz	= NULL,
788 	.sv_flags	= SV_ABI_LINUX | SV_LP64 | SV_SHP | SV_SIG_DISCIGN |
789 	    SV_SIG_WAITNDQ | SV_TIMEKEEP,
790 	.sv_set_syscall_retval = linux_set_syscall_retval,
791 	.sv_fetch_syscall_args = linux_fetch_syscall_args,
792 	.sv_syscallnames = NULL,
793 	.sv_shared_page_base = LINUX_SHAREDPAGE_LA48,
794 	.sv_shared_page_len = PAGE_SIZE,
795 	.sv_schedtail	= linux_schedtail,
796 	.sv_thread_detach = linux_thread_detach,
797 	.sv_trap	= linux_vsyscall,
798 	.sv_onexec	= linux_on_exec_vmspace,
799 	.sv_onexit	= linux_on_exit,
800 	.sv_ontdexit	= linux_thread_dtor,
801 	.sv_setid_allowed = &linux_setid_allowed_query,
802 	.sv_set_fork_retval = linux_set_fork_retval,
803 };
804 
805 static int
806 linux_on_exec_vmspace(struct proc *p, struct image_params *imgp)
807 {
808 	int error;
809 
810 	error = linux_map_vdso(p, linux_vdso_obj, linux_vdso_base,
811 	    LINUX_VDSOPAGE_SIZE, imgp);
812 	if (error == 0)
813 		linux_on_exec(p, imgp);
814 	return (error);
815 }
816 
817 /*
818  * linux_vdso_install() and linux_exec_sysvec_init() must be called
819  * after exec_sysvec_init() which is SI_SUB_EXEC (SI_ORDER_ANY).
820  */
821 static void
822 linux_exec_sysvec_init(void *param)
823 {
824 	l_uintptr_t *ktimekeep_base, *ktsc_selector;
825 	struct sysentvec *sv;
826 	ptrdiff_t tkoff;
827 
828 	sv = param;
829 	amd64_lower_shared_page(sv);
830 	/* Fill timekeep_base */
831 	exec_sysvec_init(sv);
832 
833 	tkoff = kern_timekeep_base - linux_vdso_base;
834 	ktimekeep_base = (l_uintptr_t *)(linux_vdso_mapping + tkoff);
835 	*ktimekeep_base = sv->sv_timekeep_base;
836 
837 	tkoff = kern_tsc_selector - linux_vdso_base;
838 	ktsc_selector = (l_uintptr_t *)(linux_vdso_mapping + tkoff);
839 	*ktsc_selector = linux_vdso_tsc_selector_idx();
840 	if (bootverbose)
841 		printf("Linux x86-64 vDSO tsc_selector: %lu\n", *ktsc_selector);
842 }
843 SYSINIT(elf_linux_exec_sysvec_init, SI_SUB_EXEC + 1, SI_ORDER_ANY,
844     linux_exec_sysvec_init, &elf_linux_sysvec);
845 
846 static void
847 linux_vdso_install(const void *param)
848 {
849 	char *vdso_start = &_binary_linux_vdso_so_o_start;
850 	char *vdso_end = &_binary_linux_vdso_so_o_end;
851 
852 	linux_szsigcode = vdso_end - vdso_start;
853 	MPASS(linux_szsigcode <= LINUX_VDSOPAGE_SIZE);
854 
855 	linux_vdso_base = LINUX_VDSOPAGE_LA48;
856 	if (hw_lower_amd64_sharedpage != 0)
857 		linux_vdso_base -= PAGE_SIZE;
858 
859 	__elfN(linux_vdso_fixup)(vdso_start, linux_vdso_base);
860 
861 	linux_vdso_obj = __elfN(linux_shared_page_init)
862 	    (&linux_vdso_mapping, LINUX_VDSOPAGE_SIZE);
863 	bcopy(vdso_start, linux_vdso_mapping, linux_szsigcode);
864 
865 	linux_vdso_reloc(linux_vdso_mapping, linux_vdso_base);
866 }
867 SYSINIT(elf_linux_vdso_init, SI_SUB_EXEC + 1, SI_ORDER_FIRST,
868     linux_vdso_install, NULL);
869 
870 static void
871 linux_vdso_deinstall(const void *param)
872 {
873 
874 	__elfN(linux_shared_page_fini)(linux_vdso_obj,
875 	    linux_vdso_mapping, LINUX_VDSOPAGE_SIZE);
876 }
877 SYSUNINIT(elf_linux_vdso_uninit, SI_SUB_EXEC, SI_ORDER_FIRST,
878     linux_vdso_deinstall, NULL);
879 
880 static void
881 linux_vdso_reloc(char *mapping, Elf_Addr offset)
882 {
883 	const Elf_Ehdr *ehdr;
884 	const Elf_Shdr *shdr;
885 	Elf64_Addr *where, val;
886 	Elf_Size rtype, symidx;
887 	const Elf_Rela *rela;
888 	Elf_Addr addr, addend;
889 	int relacnt;
890 	int i, j;
891 
892 	MPASS(offset != 0);
893 
894 	relacnt = 0;
895 	ehdr = (const Elf_Ehdr *)mapping;
896 	shdr = (const Elf_Shdr *)(mapping + ehdr->e_shoff);
897 	for (i = 0; i < ehdr->e_shnum; i++)
898 	{
899 		switch (shdr[i].sh_type) {
900 		case SHT_REL:
901 			printf("Linux x86_64 vDSO: unexpected Rel section\n");
902 			break;
903 		case SHT_RELA:
904 			rela = (const Elf_Rela *)(mapping + shdr[i].sh_offset);
905 			relacnt = shdr[i].sh_size / sizeof(*rela);
906 		}
907 	}
908 
909 	for (j = 0; j < relacnt; j++, rela++) {
910 		where = (Elf_Addr *)(mapping + rela->r_offset);
911 		addend = rela->r_addend;
912 		rtype = ELF_R_TYPE(rela->r_info);
913 		symidx = ELF_R_SYM(rela->r_info);
914 
915 		switch (rtype) {
916 		case R_X86_64_NONE:	/* none */
917 			break;
918 
919 		case R_X86_64_RELATIVE:	/* B + A */
920 			addr = (Elf_Addr)(offset + addend);
921 			val = addr;
922 			if (*where != val)
923 				*where = val;
924 			break;
925 		case R_X86_64_IRELATIVE:
926 			printf("Linux x86_64 vDSO: unexpected ifunc relocation, "
927 			    "symbol index %ld\n", symidx);
928 			break;
929 		default:
930 			printf("Linux x86_64 vDSO: unexpected relocation type %ld, "
931 			    "symbol index %ld\n", rtype, symidx);
932 		}
933 	}
934 }
935 
936 static char GNULINUX_ABI_VENDOR[] = "GNU";
937 static int GNULINUX_ABI_DESC = 0;
938 
939 static bool
940 linux_trans_osrel(const Elf_Note *note, int32_t *osrel)
941 {
942 	const Elf32_Word *desc;
943 	uintptr_t p;
944 
945 	p = (uintptr_t)(note + 1);
946 	p += roundup2(note->n_namesz, sizeof(Elf32_Addr));
947 
948 	desc = (const Elf32_Word *)p;
949 	if (desc[0] != GNULINUX_ABI_DESC)
950 		return (false);
951 
952 	/*
953 	 * For Linux we encode osrel using the Linux convention of
954 	 * 	(version << 16) | (major << 8) | (minor)
955 	 * See macro in linux_mib.h
956 	 */
957 	*osrel = LINUX_KERNVER(desc[1], desc[2], desc[3]);
958 
959 	return (true);
960 }
961 
962 static Elf_Brandnote linux64_brandnote = {
963 	.hdr.n_namesz	= sizeof(GNULINUX_ABI_VENDOR),
964 	.hdr.n_descsz	= 16,
965 	.hdr.n_type	= 1,
966 	.vendor		= GNULINUX_ABI_VENDOR,
967 	.flags		= BN_TRANSLATE_OSREL,
968 	.trans_osrel	= linux_trans_osrel
969 };
970 
971 static Elf64_Brandinfo linux_glibc2brand = {
972 	.brand		= ELFOSABI_LINUX,
973 	.machine	= EM_X86_64,
974 	.compat_3_brand	= "Linux",
975 	.emul_path	= linux_emul_path,
976 	.interp_path	= "/lib64/ld-linux-x86-64.so.2",
977 	.sysvec		= &elf_linux_sysvec,
978 	.interp_newpath	= NULL,
979 	.brand_note	= &linux64_brandnote,
980 	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
981 };
982 
983 static Elf64_Brandinfo linux_glibc2brandshort = {
984 	.brand		= ELFOSABI_LINUX,
985 	.machine	= EM_X86_64,
986 	.compat_3_brand	= "Linux",
987 	.emul_path	= linux_emul_path,
988 	.interp_path	= "/lib64/ld-linux.so.2",
989 	.sysvec		= &elf_linux_sysvec,
990 	.interp_newpath	= NULL,
991 	.brand_note	= &linux64_brandnote,
992 	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE
993 };
994 
995 static Elf64_Brandinfo linux_muslbrand = {
996 	.brand		= ELFOSABI_LINUX,
997 	.machine	= EM_X86_64,
998 	.compat_3_brand	= "Linux",
999 	.emul_path	= linux_emul_path,
1000 	.interp_path	= "/lib/ld-musl-x86_64.so.1",
1001 	.sysvec		= &elf_linux_sysvec,
1002 	.interp_newpath	= NULL,
1003 	.brand_note	= &linux64_brandnote,
1004 	.flags		= BI_CAN_EXEC_DYN | BI_BRAND_NOTE |
1005 			    LINUX_BI_FUTEX_REQUEUE
1006 };
1007 
1008 Elf64_Brandinfo *linux_brandlist[] = {
1009 	&linux_glibc2brand,
1010 	&linux_glibc2brandshort,
1011 	&linux_muslbrand,
1012 	NULL
1013 };
1014 
1015 static int
1016 linux64_elf_modevent(module_t mod, int type, void *data)
1017 {
1018 	Elf64_Brandinfo **brandinfo;
1019 	int error;
1020 	struct linux_ioctl_handler **lihp;
1021 
1022 	error = 0;
1023 
1024 	switch(type) {
1025 	case MOD_LOAD:
1026 		for (brandinfo = &linux_brandlist[0]; *brandinfo != NULL;
1027 		     ++brandinfo)
1028 			if (elf64_insert_brand_entry(*brandinfo) < 0)
1029 				error = EINVAL;
1030 		if (error == 0) {
1031 			SET_FOREACH(lihp, linux_ioctl_handler_set)
1032 				linux_ioctl_register_handler(*lihp);
1033 			stclohz = (stathz ? stathz : hz);
1034 			if (bootverbose)
1035 				printf("Linux x86-64 ELF exec handler installed\n");
1036 		} else
1037 			printf("cannot insert Linux x86-64 ELF brand handler\n");
1038 		break;
1039 	case MOD_UNLOAD:
1040 		for (brandinfo = &linux_brandlist[0]; *brandinfo != NULL;
1041 		     ++brandinfo)
1042 			if (elf64_brand_inuse(*brandinfo))
1043 				error = EBUSY;
1044 		if (error == 0) {
1045 			for (brandinfo = &linux_brandlist[0];
1046 			     *brandinfo != NULL; ++brandinfo)
1047 				if (elf64_remove_brand_entry(*brandinfo) < 0)
1048 					error = EINVAL;
1049 		}
1050 		if (error == 0) {
1051 			SET_FOREACH(lihp, linux_ioctl_handler_set)
1052 				linux_ioctl_unregister_handler(*lihp);
1053 			if (bootverbose)
1054 				printf("Linux x86_64 ELF exec handler removed\n");
1055 		} else
1056 			printf("Could not deinstall Linux x86_64 ELF interpreter entry\n");
1057 		break;
1058 	default:
1059 		return (EOPNOTSUPP);
1060 	}
1061 	return (error);
1062 }
1063 
1064 static moduledata_t linux64_elf_mod = {
1065 	"linux64elf",
1066 	linux64_elf_modevent,
1067 	0
1068 };
1069 
1070 DECLARE_MODULE_TIED(linux64elf, linux64_elf_mod, SI_SUB_EXEC, SI_ORDER_ANY);
1071 MODULE_DEPEND(linux64elf, linux_common, 1, 1, 1);
1072 FEATURE(linux64, "Linux 64bit support");
1073