1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */
27 /* All Rights Reserved */
28
29 #include <sys/types.h>
30 #include <sys/param.h>
31 #include <sys/sysmacros.h>
32 #include <sys/signal.h>
33 #include <sys/systm.h>
34 #include <sys/user.h>
35 #include <sys/mman.h>
36 #include <sys/class.h>
37 #include <sys/proc.h>
38 #include <sys/procfs.h>
39 #include <sys/buf.h>
40 #include <sys/kmem.h>
41 #include <sys/cred.h>
42 #include <sys/archsystm.h>
43 #include <sys/vmparam.h>
44 #include <sys/prsystm.h>
45 #include <sys/reboot.h>
46 #include <sys/uadmin.h>
47 #include <sys/vfs.h>
48 #include <sys/vnode.h>
49 #include <sys/file.h>
50 #include <sys/session.h>
51 #include <sys/ucontext.h>
52 #include <sys/dnlc.h>
53 #include <sys/var.h>
54 #include <sys/cmn_err.h>
55 #include <sys/debugreg.h>
56 #include <sys/thread.h>
57 #include <sys/vtrace.h>
58 #include <sys/consdev.h>
59 #include <sys/psw.h>
60 #include <sys/regset.h>
61 #include <sys/privregs.h>
62 #include <sys/cpu.h>
63 #include <sys/stack.h>
64 #include <sys/swap.h>
65 #include <vm/hat.h>
66 #include <vm/anon.h>
67 #include <vm/as.h>
68 #include <vm/page.h>
69 #include <vm/seg.h>
70 #include <vm/seg_kmem.h>
71 #include <vm/seg_map.h>
72 #include <vm/seg_vn.h>
73 #include <sys/exec.h>
74 #include <sys/acct.h>
75 #include <sys/core.h>
76 #include <sys/corectl.h>
77 #include <sys/modctl.h>
78 #include <sys/tuneable.h>
79 #include <c2/audit.h>
80 #include <sys/bootconf.h>
81 #include <sys/brand.h>
82 #include <sys/dumphdr.h>
83 #include <sys/promif.h>
84 #include <sys/systeminfo.h>
85 #include <sys/kdi.h>
86 #include <sys/contract_impl.h>
87 #include <sys/x86_archext.h>
88 #include <sys/segments.h>
89 #include <sys/ontrap.h>
90 #include <sys/cpu.h>
91 #ifdef __xpv
92 #include <sys/hypervisor.h>
93 #endif
94
95 /*
96 * Compare the version of boot that boot says it is against
97 * the version of boot the kernel expects.
98 */
99 int
check_boot_version(int boots_version)100 check_boot_version(int boots_version)
101 {
102 if (boots_version == BO_VERSION)
103 return (0);
104
105 prom_printf("Wrong boot interface - kernel needs v%d found v%d\n",
106 BO_VERSION, boots_version);
107 prom_panic("halting");
108 /*NOTREACHED*/
109 }
110
111 /*
112 * Process the physical installed list for boot.
113 * Finds:
114 * 1) the pfn of the highest installed physical page,
115 * 2) the number of pages installed
116 * 3) the number of distinct contiguous regions these pages fall into.
117 * 4) the number of contiguous memory ranges
118 */
119 void
installed_top_size_ex(struct memlist * list,pfn_t * high_pfn,pgcnt_t * pgcnt,int * ranges)120 installed_top_size_ex(
121 struct memlist *list, /* pointer to start of installed list */
122 pfn_t *high_pfn, /* return ptr for top value */
123 pgcnt_t *pgcnt, /* return ptr for sum of installed pages */
124 int *ranges) /* return ptr for the count of contig. ranges */
125 {
126 pfn_t top = 0;
127 pgcnt_t sumpages = 0;
128 pfn_t highp; /* high page in a chunk */
129 int cnt = 0;
130
131 for (; list; list = list->ml_next) {
132 ++cnt;
133 highp = (list->ml_address + list->ml_size - 1) >> PAGESHIFT;
134 if (top < highp)
135 top = highp;
136 sumpages += btop(list->ml_size);
137 }
138
139 *high_pfn = top;
140 *pgcnt = sumpages;
141 *ranges = cnt;
142 }
143
144 void
installed_top_size(struct memlist * list,pfn_t * high_pfn,pgcnt_t * pgcnt)145 installed_top_size(
146 struct memlist *list, /* pointer to start of installed list */
147 pfn_t *high_pfn, /* return ptr for top value */
148 pgcnt_t *pgcnt) /* return ptr for sum of installed pages */
149 {
150 int ranges;
151
152 installed_top_size_ex(list, high_pfn, pgcnt, &ranges);
153 }
154
155 void
phys_install_has_changed(void)156 phys_install_has_changed(void)
157 {}
158
159 /*
160 * Copy in a memory list from boot to kernel, with a filter function
161 * to remove pages. The filter function can increase the address and/or
162 * decrease the size to filter out pages. It will also align addresses and
163 * sizes to PAGESIZE.
164 */
165 void
copy_memlist_filter(struct memlist * src,struct memlist ** dstp,void (* filter)(uint64_t *,uint64_t *))166 copy_memlist_filter(
167 struct memlist *src,
168 struct memlist **dstp,
169 void (*filter)(uint64_t *, uint64_t *))
170 {
171 struct memlist *dst, *prev;
172 uint64_t addr;
173 uint64_t size;
174 uint64_t eaddr;
175
176 dst = *dstp;
177 prev = dst;
178
179 /*
180 * Move through the memlist applying a filter against
181 * each range of memory. Note that we may apply the
182 * filter multiple times against each memlist entry.
183 */
184 for (; src; src = src->ml_next) {
185 addr = P2ROUNDUP(src->ml_address, PAGESIZE);
186 eaddr = P2ALIGN(src->ml_address + src->ml_size, PAGESIZE);
187 while (addr < eaddr) {
188 size = eaddr - addr;
189 if (filter != NULL)
190 filter(&addr, &size);
191 if (size == 0)
192 break;
193 dst->ml_address = addr;
194 dst->ml_size = size;
195 dst->ml_next = 0;
196 if (prev == dst) {
197 dst->ml_prev = 0;
198 dst++;
199 } else {
200 dst->ml_prev = prev;
201 prev->ml_next = dst;
202 dst++;
203 prev++;
204 }
205 addr += size;
206 }
207 }
208
209 *dstp = dst;
210 }
211
212 /*
213 * Kernel setup code, called from startup().
214 */
215 void
kern_setup1(void)216 kern_setup1(void)
217 {
218 proc_t *pp;
219
220 pp = &p0;
221
222 proc_sched = pp;
223
224 /*
225 * Initialize process 0 data structures
226 */
227 pp->p_stat = SRUN;
228 pp->p_flag = SSYS;
229
230 pp->p_pidp = &pid0;
231 pp->p_pgidp = &pid0;
232 pp->p_sessp = &session0;
233 pp->p_tlist = &t0;
234 pid0.pid_pglink = pp;
235 pid0.pid_pgtail = pp;
236
237 /*
238 * XXX - we asssume that the u-area is zeroed out except for
239 * ttolwp(curthread)->lwp_regs.
240 */
241 PTOU(curproc)->u_cmask = (mode_t)CMASK;
242
243 thread_init(); /* init thread_free list */
244 pid_init(); /* initialize pid (proc) table */
245 contract_init(); /* initialize contracts */
246
247 init_pages_pp_maximum();
248 }
249
250 /*
251 * Load a procedure into a thread.
252 */
253 void
thread_load(kthread_t * t,void (* start)(),caddr_t arg,size_t len)254 thread_load(kthread_t *t, void (*start)(), caddr_t arg, size_t len)
255 {
256 caddr_t sp;
257 size_t framesz;
258 caddr_t argp;
259 long *p;
260 extern void thread_start();
261
262 /*
263 * Push a "c" call frame onto the stack to represent
264 * the caller of "start".
265 */
266 sp = t->t_stk;
267 ASSERT(((uintptr_t)t->t_stk & (STACK_ENTRY_ALIGN - 1)) == 0);
268 if (len != 0) {
269 /*
270 * the object that arg points at is copied into the
271 * caller's frame.
272 */
273 framesz = SA(len);
274 sp -= framesz;
275 ASSERT(sp > t->t_stkbase);
276 argp = sp + SA(MINFRAME);
277 bcopy(arg, argp, len);
278 arg = argp;
279 }
280 /*
281 * Set up arguments (arg and len) on the caller's stack frame.
282 */
283 p = (long *)sp;
284
285 *--p = 0; /* fake call */
286 *--p = 0; /* null frame pointer terminates stack trace */
287 *--p = (long)len;
288 *--p = (intptr_t)arg;
289 *--p = (intptr_t)start;
290
291 /*
292 * initialize thread to resume at thread_start() which will
293 * turn around and invoke (*start)(arg, len).
294 */
295 t->t_pc = (uintptr_t)thread_start;
296 t->t_sp = (uintptr_t)p;
297
298 ASSERT((t->t_sp & (STACK_ENTRY_ALIGN - 1)) == 0);
299 }
300
301 /*
302 * load user registers into lwp.
303 */
304 /*ARGSUSED2*/
305 void
lwp_load(klwp_t * lwp,gregset_t grp,uintptr_t thrptr)306 lwp_load(klwp_t *lwp, gregset_t grp, uintptr_t thrptr)
307 {
308 struct regs *rp = lwptoregs(lwp);
309
310 setgregs(lwp, grp);
311 rp->r_ps = PSL_USER;
312
313 /*
314 * For 64-bit lwps, we allow one magic %fs selector value, and one
315 * magic %gs selector to point anywhere in the address space using
316 * %fsbase and %gsbase behind the scenes. libc uses %fs to point
317 * at the ulwp_t structure.
318 *
319 * For 32-bit lwps, libc wedges its lwp thread pointer into the
320 * ucontext ESP slot (which is otherwise irrelevant to setting a
321 * ucontext) and LWPGS_SEL value into gregs[REG_GS]. This is so
322 * syslwp_create() can atomically setup %gs.
323 *
324 * See setup_context() in libc.
325 */
326 #ifdef _SYSCALL32_IMPL
327 if (lwp_getdatamodel(lwp) == DATAMODEL_ILP32) {
328 if (grp[REG_GS] == LWPGS_SEL)
329 (void) lwp_setprivate(lwp, _LWP_GSBASE, thrptr);
330 } else {
331 /*
332 * See lwp_setprivate in kernel and setup_context in libc.
333 *
334 * Currently libc constructs a ucontext from whole cloth for
335 * every new (not main) lwp created. For 64 bit processes
336 * %fsbase is directly set to point to current thread pointer.
337 * In the past (solaris 10) %fs was also set LWPFS_SEL to
338 * indicate %fsbase. Now we use the null GDT selector for
339 * this purpose. LWP[FS|GS]_SEL are only intended for 32 bit
340 * processes. To ease transition we support older libcs in
341 * the newer kernel by forcing %fs or %gs selector to null
342 * by calling lwp_setprivate if LWP[FS|GS]_SEL is passed in
343 * the ucontext. This is should be ripped out at some future
344 * date. Another fix would be for libc to do a getcontext
345 * and inherit the null %fs/%gs from the current context but
346 * that means an extra system call and could hurt performance.
347 */
348 if (grp[REG_FS] == 0x1bb) /* hard code legacy LWPFS_SEL */
349 (void) lwp_setprivate(lwp, _LWP_FSBASE,
350 (uintptr_t)grp[REG_FSBASE]);
351
352 if (grp[REG_GS] == 0x1c3) /* hard code legacy LWPGS_SEL */
353 (void) lwp_setprivate(lwp, _LWP_GSBASE,
354 (uintptr_t)grp[REG_GSBASE]);
355 }
356 #else
357 if (grp[GS] == LWPGS_SEL)
358 (void) lwp_setprivate(lwp, _LWP_GSBASE, thrptr);
359 #endif
360
361 lwp->lwp_eosys = JUSTRETURN;
362 lwptot(lwp)->t_post_sys = 1;
363 }
364
365 /*
366 * set syscall()'s return values for a lwp.
367 */
368 void
lwp_setrval(klwp_t * lwp,int v1,int v2)369 lwp_setrval(klwp_t *lwp, int v1, int v2)
370 {
371 lwptoregs(lwp)->r_ps &= ~PS_C;
372 lwptoregs(lwp)->r_r0 = v1;
373 lwptoregs(lwp)->r_r1 = v2;
374 }
375
376 /*
377 * set syscall()'s return values for a lwp.
378 */
379 void
lwp_setsp(klwp_t * lwp,caddr_t sp)380 lwp_setsp(klwp_t *lwp, caddr_t sp)
381 {
382 lwptoregs(lwp)->r_sp = (intptr_t)sp;
383 }
384
385 /*
386 * Copy regs from parent to child.
387 */
388 void
lwp_forkregs(klwp_t * lwp,klwp_t * clwp)389 lwp_forkregs(klwp_t *lwp, klwp_t *clwp)
390 {
391 #if defined(__amd64)
392 struct pcb *pcb = &clwp->lwp_pcb;
393 struct regs *rp = lwptoregs(lwp);
394
395 if (pcb->pcb_rupdate == 0) {
396 pcb->pcb_ds = rp->r_ds;
397 pcb->pcb_es = rp->r_es;
398 pcb->pcb_fs = rp->r_fs;
399 pcb->pcb_gs = rp->r_gs;
400 pcb->pcb_rupdate = 1;
401 lwptot(clwp)->t_post_sys = 1;
402 }
403 ASSERT(lwptot(clwp)->t_post_sys);
404 #endif
405
406 bcopy(lwp->lwp_regs, clwp->lwp_regs, sizeof (struct regs));
407 }
408
409 /*
410 * This function is currently unused on x86.
411 */
412 /*ARGSUSED*/
413 void
lwp_freeregs(klwp_t * lwp,int isexec)414 lwp_freeregs(klwp_t *lwp, int isexec)
415 {}
416
417 /*
418 * This function is currently unused on x86.
419 */
420 void
lwp_pcb_exit(void)421 lwp_pcb_exit(void)
422 {}
423
424 /*
425 * Lwp context ops for segment registers.
426 */
427
428 /*
429 * Every time we come into the kernel (syscall, interrupt or trap
430 * but not fast-traps) we capture the current values of the user's
431 * segment registers into the lwp's reg structure. This includes
432 * lcall for i386 generic system call support since it is handled
433 * as a segment-not-present trap.
434 *
435 * Here we save the current values from the lwp regs into the pcb
436 * and set pcb->pcb_rupdate to 1 to tell the rest of the kernel
437 * that the pcb copy of the segment registers is the current one.
438 * This ensures the lwp's next trip to user land via update_sregs.
439 * Finally we set t_post_sys to ensure that no system call fast-path's
440 * its way out of the kernel via sysret.
441 *
442 * (This means that we need to have interrupts disabled when we test
443 * t->t_post_sys in the syscall handlers; if the test fails, we need
444 * to keep interrupts disabled until we return to userland so we can't
445 * be switched away.)
446 *
447 * As a result of all this, we don't really have to do a whole lot if
448 * the thread is just mucking about in the kernel, switching on and
449 * off the cpu for whatever reason it feels like. And yet we still
450 * preserve fast syscalls, cause if we -don't- get descheduled,
451 * we never come here either.
452 */
453
454 #define VALID_LWP_DESC(udp) ((udp)->usd_type == SDT_MEMRWA && \
455 (udp)->usd_p == 1 && (udp)->usd_dpl == SEL_UPL)
456
457 /*ARGSUSED*/
458 void
lwp_segregs_save(klwp_t * lwp)459 lwp_segregs_save(klwp_t *lwp)
460 {
461 #if defined(__amd64)
462 pcb_t *pcb = &lwp->lwp_pcb;
463 struct regs *rp;
464
465 ASSERT(VALID_LWP_DESC(&pcb->pcb_fsdesc));
466 ASSERT(VALID_LWP_DESC(&pcb->pcb_gsdesc));
467
468 if (pcb->pcb_rupdate == 0) {
469 rp = lwptoregs(lwp);
470
471 /*
472 * If there's no update already pending, capture the current
473 * %ds/%es/%fs/%gs values from lwp's regs in case the user
474 * changed them; %fsbase and %gsbase are privileged so the
475 * kernel versions of these registers in pcb_fsbase and
476 * pcb_gsbase are always up-to-date.
477 */
478 pcb->pcb_ds = rp->r_ds;
479 pcb->pcb_es = rp->r_es;
480 pcb->pcb_fs = rp->r_fs;
481 pcb->pcb_gs = rp->r_gs;
482 pcb->pcb_rupdate = 1;
483 lwp->lwp_thread->t_post_sys = 1;
484 }
485 #endif /* __amd64 */
486
487 #if !defined(__xpv) /* XXPV not sure if we can re-read gdt? */
488 ASSERT(bcmp(&CPU->cpu_gdt[GDT_LWPFS], &lwp->lwp_pcb.pcb_fsdesc,
489 sizeof (lwp->lwp_pcb.pcb_fsdesc)) == 0);
490 ASSERT(bcmp(&CPU->cpu_gdt[GDT_LWPGS], &lwp->lwp_pcb.pcb_gsdesc,
491 sizeof (lwp->lwp_pcb.pcb_gsdesc)) == 0);
492 #endif
493 }
494
495 #if defined(__amd64)
496
497 /*
498 * Update the segment registers with new values from the pcb.
499 *
500 * We have to do this carefully, and in the following order,
501 * in case any of the selectors points at a bogus descriptor.
502 * If they do, we'll catch trap with on_trap and return 1.
503 * returns 0 on success.
504 *
505 * This is particularly tricky for %gs.
506 * This routine must be executed under a cli.
507 */
508 int
update_sregs(struct regs * rp,klwp_t * lwp)509 update_sregs(struct regs *rp, klwp_t *lwp)
510 {
511 pcb_t *pcb = &lwp->lwp_pcb;
512 ulong_t kgsbase;
513 on_trap_data_t otd;
514 int rc = 0;
515
516 if (!on_trap(&otd, OT_SEGMENT_ACCESS)) {
517
518 #if defined(__xpv)
519 /*
520 * On the hyervisor this is easy. The hypercall below will
521 * swapgs and load %gs with the user selector. If the user
522 * selector is bad the hypervisor will catch the fault and
523 * load %gs with the null selector instead. Either way the
524 * kernel's gsbase is not damaged.
525 */
526 kgsbase = (ulong_t)CPU;
527 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL,
528 pcb->pcb_gs) != 0) {
529 no_trap();
530 return (1);
531 }
532
533 rp->r_gs = pcb->pcb_gs;
534 ASSERT((cpu_t *)kgsbase == CPU);
535
536 #else /* __xpv */
537
538 /*
539 * A little more complicated running native.
540 */
541 kgsbase = (ulong_t)CPU;
542 __set_gs(pcb->pcb_gs);
543
544 /*
545 * If __set_gs fails it's because the new %gs is a bad %gs,
546 * we'll be taking a trap but with the original %gs and %gsbase
547 * undamaged (i.e. pointing at curcpu).
548 *
549 * We've just mucked up the kernel's gsbase. Oops. In
550 * particular we can't take any traps at all. Make the newly
551 * computed gsbase be the hidden gs via __swapgs, and fix
552 * the kernel's gsbase back again. Later, when we return to
553 * userland we'll swapgs again restoring gsbase just loaded
554 * above.
555 */
556 __swapgs();
557 rp->r_gs = pcb->pcb_gs;
558
559 /*
560 * restore kernel's gsbase
561 */
562 wrmsr(MSR_AMD_GSBASE, kgsbase);
563
564 #endif /* __xpv */
565
566 /*
567 * Only override the descriptor base address if
568 * r_gs == LWPGS_SEL or if r_gs == NULL. A note on
569 * NULL descriptors -- 32-bit programs take faults
570 * if they deference NULL descriptors; however,
571 * when 64-bit programs load them into %fs or %gs,
572 * they DONT fault -- only the base address remains
573 * whatever it was from the last load. Urk.
574 *
575 * XXX - note that lwp_setprivate now sets %fs/%gs to the
576 * null selector for 64 bit processes. Whereas before
577 * %fs/%gs were set to LWP(FS|GS)_SEL regardless of
578 * the process's data model. For now we check for both
579 * values so that the kernel can also support the older
580 * libc. This should be ripped out at some point in the
581 * future.
582 */
583 if (pcb->pcb_gs == LWPGS_SEL || pcb->pcb_gs == 0) {
584 #if defined(__xpv)
585 if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER,
586 pcb->pcb_gsbase)) {
587 no_trap();
588 return (1);
589 }
590 #else
591 wrmsr(MSR_AMD_KGSBASE, pcb->pcb_gsbase);
592 #endif
593 }
594
595 __set_ds(pcb->pcb_ds);
596 rp->r_ds = pcb->pcb_ds;
597
598 __set_es(pcb->pcb_es);
599 rp->r_es = pcb->pcb_es;
600
601 __set_fs(pcb->pcb_fs);
602 rp->r_fs = pcb->pcb_fs;
603
604 /*
605 * Same as for %gs
606 */
607 if (pcb->pcb_fs == LWPFS_SEL || pcb->pcb_fs == 0) {
608 #if defined(__xpv)
609 if (HYPERVISOR_set_segment_base(SEGBASE_FS,
610 pcb->pcb_fsbase)) {
611 no_trap();
612 return (1);
613 }
614 #else
615 wrmsr(MSR_AMD_FSBASE, pcb->pcb_fsbase);
616 #endif
617 }
618
619 } else {
620 cli();
621 rc = 1;
622 }
623 no_trap();
624 return (rc);
625 }
626
627 /*
628 * Make sure any stale selectors are cleared from the segment registers
629 * by putting KDS_SEL (the kernel's default %ds gdt selector) into them.
630 * This is necessary because the kernel itself does not use %es, %fs, nor
631 * %ds. (%cs and %ss are necessary, and are set up by the kernel - along with
632 * %gs - to point to the current cpu struct.) If we enter kmdb while in the
633 * kernel and resume with a stale ldt or brandz selector sitting there in a
634 * segment register, kmdb will #gp fault if the stale selector points to,
635 * for example, an ldt in the context of another process.
636 *
637 * WARNING: Intel and AMD chips behave differently when storing
638 * the null selector into %fs and %gs while in long mode. On AMD
639 * chips fsbase and gsbase are not cleared. But on Intel chips, storing
640 * a null selector into %fs or %gs has the side effect of clearing
641 * fsbase or gsbase. For that reason we use KDS_SEL, which has
642 * consistent behavor between AMD and Intel.
643 *
644 * Caller responsible for preventing cpu migration.
645 */
646 void
reset_sregs(void)647 reset_sregs(void)
648 {
649 ulong_t kgsbase = (ulong_t)CPU;
650
651 ASSERT(curthread->t_preempt != 0 || getpil() >= DISP_LEVEL);
652
653 cli();
654 __set_gs(KGS_SEL);
655
656 /*
657 * restore kernel gsbase
658 */
659 #if defined(__xpv)
660 xen_set_segment_base(SEGBASE_GS_KERNEL, kgsbase);
661 #else
662 wrmsr(MSR_AMD_GSBASE, kgsbase);
663 #endif
664
665 sti();
666
667 __set_ds(KDS_SEL);
668 __set_es(0 | SEL_KPL); /* selector RPL not ring 0 on hypervisor */
669 __set_fs(KFS_SEL);
670 }
671
672 #endif /* __amd64 */
673
674 #ifdef _SYSCALL32_IMPL
675
676 /*
677 * Make it impossible for a process to change its data model.
678 * We do this by toggling the present bits for the 32 and
679 * 64-bit user code descriptors. That way if a user lwp attempts
680 * to change its data model (by using the wrong code descriptor in
681 * %cs) it will fault immediately. This also allows us to simplify
682 * assertions and checks in the kernel.
683 */
684
685 static void
gdt_ucode_model(model_t model)686 gdt_ucode_model(model_t model)
687 {
688 kpreempt_disable();
689 if (model == DATAMODEL_NATIVE) {
690 gdt_update_usegd(GDT_UCODE, &ucs_on);
691 gdt_update_usegd(GDT_U32CODE, &ucs32_off);
692 } else {
693 gdt_update_usegd(GDT_U32CODE, &ucs32_on);
694 gdt_update_usegd(GDT_UCODE, &ucs_off);
695 }
696 kpreempt_enable();
697 }
698
699 #endif /* _SYSCALL32_IMPL */
700
701 /*
702 * Restore lwp private fs and gs segment descriptors
703 * on current cpu's GDT.
704 */
705 static void
lwp_segregs_restore(klwp_t * lwp)706 lwp_segregs_restore(klwp_t *lwp)
707 {
708 pcb_t *pcb = &lwp->lwp_pcb;
709
710 ASSERT(VALID_LWP_DESC(&pcb->pcb_fsdesc));
711 ASSERT(VALID_LWP_DESC(&pcb->pcb_gsdesc));
712
713 #ifdef _SYSCALL32_IMPL
714 gdt_ucode_model(DATAMODEL_NATIVE);
715 #endif
716
717 gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc);
718 gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc);
719
720 }
721
722 #ifdef _SYSCALL32_IMPL
723
724 static void
lwp_segregs_restore32(klwp_t * lwp)725 lwp_segregs_restore32(klwp_t *lwp)
726 {
727 /*LINTED*/
728 cpu_t *cpu = CPU;
729 pcb_t *pcb = &lwp->lwp_pcb;
730
731 ASSERT(VALID_LWP_DESC(&lwp->lwp_pcb.pcb_fsdesc));
732 ASSERT(VALID_LWP_DESC(&lwp->lwp_pcb.pcb_gsdesc));
733
734 gdt_ucode_model(DATAMODEL_ILP32);
735 gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc);
736 gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc);
737 }
738
739 #endif /* _SYSCALL32_IMPL */
740
741 /*
742 * If this is a process in a branded zone, then we want it to use the brand
743 * syscall entry points instead of the standard Solaris entry points. This
744 * routine must be called when a new lwp is created within a branded zone
745 * or when an existing lwp moves into a branded zone via a zone_enter()
746 * operation.
747 */
748 void
lwp_attach_brand_hdlrs(klwp_t * lwp)749 lwp_attach_brand_hdlrs(klwp_t *lwp)
750 {
751 kthread_t *t = lwptot(lwp);
752
753 ASSERT(PROC_IS_BRANDED(lwptoproc(lwp)));
754
755 ASSERT(removectx(t, NULL, brand_interpositioning_disable,
756 brand_interpositioning_enable, NULL, NULL,
757 brand_interpositioning_disable, NULL) == 0);
758 installctx(t, NULL, brand_interpositioning_disable,
759 brand_interpositioning_enable, NULL, NULL,
760 brand_interpositioning_disable, NULL);
761
762 if (t == curthread) {
763 kpreempt_disable();
764 brand_interpositioning_enable();
765 kpreempt_enable();
766 }
767 }
768
769 /*
770 * If this is a process in a branded zone, then we want it to disable the
771 * brand syscall entry points. This routine must be called when the last
772 * lwp in a process is exiting in proc_exit().
773 */
774 void
lwp_detach_brand_hdlrs(klwp_t * lwp)775 lwp_detach_brand_hdlrs(klwp_t *lwp)
776 {
777 kthread_t *t = lwptot(lwp);
778
779 ASSERT(PROC_IS_BRANDED(lwptoproc(lwp)));
780 if (t == curthread)
781 kpreempt_disable();
782
783 /* Remove the original context handlers */
784 VERIFY(removectx(t, NULL, brand_interpositioning_disable,
785 brand_interpositioning_enable, NULL, NULL,
786 brand_interpositioning_disable, NULL) != 0);
787
788 if (t == curthread) {
789 /* Cleanup our MSR and IDT entries. */
790 brand_interpositioning_disable();
791 kpreempt_enable();
792 }
793 }
794
795 /*
796 * Add any lwp-associated context handlers to the lwp at the beginning
797 * of the lwp's useful life.
798 *
799 * All paths which create lwp's invoke lwp_create(); lwp_create()
800 * invokes lwp_stk_init() which initializes the stack, sets up
801 * lwp_regs, and invokes this routine.
802 *
803 * All paths which destroy lwp's invoke lwp_exit() to rip the lwp
804 * apart and put it on 'lwp_deathrow'; if the lwp is destroyed it
805 * ends up in thread_free() which invokes freectx(t, 0) before
806 * invoking lwp_stk_fini(). When the lwp is recycled from death
807 * row, lwp_stk_fini() is invoked, then thread_free(), and thus
808 * freectx(t, 0) as before.
809 *
810 * In the case of exec, the surviving lwp is thoroughly scrubbed
811 * clean; exec invokes freectx(t, 1) to destroy associated contexts.
812 * On the way back to the new image, it invokes setregs() which
813 * in turn invokes this routine.
814 */
815 void
lwp_installctx(klwp_t * lwp)816 lwp_installctx(klwp_t *lwp)
817 {
818 kthread_t *t = lwptot(lwp);
819 int thisthread = t == curthread;
820 #ifdef _SYSCALL32_IMPL
821 void (*restop)(klwp_t *) = lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ?
822 lwp_segregs_restore : lwp_segregs_restore32;
823 #else
824 void (*restop)(klwp_t *) = lwp_segregs_restore;
825 #endif
826
827 /*
828 * Install the basic lwp context handlers on each lwp.
829 *
830 * On the amd64 kernel, the context handlers are responsible for
831 * virtualizing %ds, %es, %fs, and %gs to the lwp. The register
832 * values are only ever changed via sys_rtt when the
833 * pcb->pcb_rupdate == 1. Only sys_rtt gets to clear the bit.
834 *
835 * On the i386 kernel, the context handlers are responsible for
836 * virtualizing %gs/%fs to the lwp by updating the per-cpu GDTs
837 */
838 ASSERT(removectx(t, lwp, lwp_segregs_save, restop,
839 NULL, NULL, NULL, NULL) == 0);
840 if (thisthread)
841 kpreempt_disable();
842 installctx(t, lwp, lwp_segregs_save, restop,
843 NULL, NULL, NULL, NULL);
844 if (thisthread) {
845 /*
846 * Since we're the right thread, set the values in the GDT
847 */
848 restop(lwp);
849 kpreempt_enable();
850 }
851
852 /*
853 * If we have sysenter/sysexit instructions enabled, we need
854 * to ensure that the hardware mechanism is kept up-to-date with the
855 * lwp's kernel stack pointer across context switches.
856 *
857 * sep_save zeros the sysenter stack pointer msr; sep_restore sets
858 * it to the lwp's kernel stack pointer (kstktop).
859 */
860 if (is_x86_feature(x86_featureset, X86FSET_SEP)) {
861 #if defined(__amd64)
862 caddr_t kstktop = (caddr_t)lwp->lwp_regs;
863 #elif defined(__i386)
864 caddr_t kstktop = ((caddr_t)lwp->lwp_regs - MINFRAME) +
865 SA(sizeof (struct regs) + MINFRAME);
866 #endif
867 ASSERT(removectx(t, kstktop,
868 sep_save, sep_restore, NULL, NULL, NULL, NULL) == 0);
869
870 if (thisthread)
871 kpreempt_disable();
872 installctx(t, kstktop,
873 sep_save, sep_restore, NULL, NULL, NULL, NULL);
874 if (thisthread) {
875 /*
876 * We're the right thread, so set the stack pointer
877 * for the first sysenter instruction to use
878 */
879 sep_restore(kstktop);
880 kpreempt_enable();
881 }
882 }
883
884 if (PROC_IS_BRANDED(ttoproc(t)))
885 lwp_attach_brand_hdlrs(lwp);
886 }
887
888 /*
889 * Clear registers on exec(2).
890 */
891 void
setregs(uarg_t * args)892 setregs(uarg_t *args)
893 {
894 struct regs *rp;
895 kthread_t *t = curthread;
896 klwp_t *lwp = ttolwp(t);
897 pcb_t *pcb = &lwp->lwp_pcb;
898 greg_t sp;
899
900 /*
901 * Initialize user registers
902 */
903 (void) save_syscall_args(); /* copy args from registers first */
904 rp = lwptoregs(lwp);
905 sp = rp->r_sp;
906 bzero(rp, sizeof (*rp));
907
908 rp->r_ss = UDS_SEL;
909 rp->r_sp = sp;
910 rp->r_pc = args->entry;
911 rp->r_ps = PSL_USER;
912
913 #if defined(__amd64)
914
915 pcb->pcb_fs = pcb->pcb_gs = 0;
916 pcb->pcb_fsbase = pcb->pcb_gsbase = 0;
917
918 if (ttoproc(t)->p_model == DATAMODEL_NATIVE) {
919
920 rp->r_cs = UCS_SEL;
921
922 /*
923 * Only allow 64-bit user code descriptor to be present.
924 */
925 gdt_ucode_model(DATAMODEL_NATIVE);
926
927 /*
928 * Arrange that the virtualized %fs and %gs GDT descriptors
929 * have a well-defined initial state (present, ring 3
930 * and of type data).
931 */
932 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
933
934 /*
935 * thrptr is either NULL or a value used by DTrace.
936 * 64-bit processes use %fs as their "thread" register.
937 */
938 if (args->thrptr)
939 (void) lwp_setprivate(lwp, _LWP_FSBASE, args->thrptr);
940
941 } else {
942
943 rp->r_cs = U32CS_SEL;
944 rp->r_ds = rp->r_es = UDS_SEL;
945
946 /*
947 * only allow 32-bit user code selector to be present.
948 */
949 gdt_ucode_model(DATAMODEL_ILP32);
950
951 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
952
953 /*
954 * thrptr is either NULL or a value used by DTrace.
955 * 32-bit processes use %gs as their "thread" register.
956 */
957 if (args->thrptr)
958 (void) lwp_setprivate(lwp, _LWP_GSBASE, args->thrptr);
959
960 }
961
962 pcb->pcb_ds = rp->r_ds;
963 pcb->pcb_es = rp->r_es;
964 pcb->pcb_rupdate = 1;
965
966 #elif defined(__i386)
967
968 rp->r_cs = UCS_SEL;
969 rp->r_ds = rp->r_es = UDS_SEL;
970
971 /*
972 * Arrange that the virtualized %fs and %gs GDT descriptors
973 * have a well-defined initial state (present, ring 3
974 * and of type data).
975 */
976 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
977
978 /*
979 * For %gs we need to reset LWP_GSBASE in pcb and the
980 * per-cpu GDT descriptor. thrptr is either NULL
981 * or a value used by DTrace.
982 */
983 if (args->thrptr)
984 (void) lwp_setprivate(lwp, _LWP_GSBASE, args->thrptr);
985 #endif
986
987 lwp->lwp_eosys = JUSTRETURN;
988 t->t_post_sys = 1;
989
990 /*
991 * Here we initialize minimal fpu state.
992 * The rest is done at the first floating
993 * point instruction that a process executes.
994 */
995 pcb->pcb_fpu.fpu_flags = 0;
996
997 /*
998 * Add the lwp context handlers that virtualize segment registers,
999 * and/or system call stacks etc.
1000 */
1001 lwp_installctx(lwp);
1002 }
1003
1004 user_desc_t *
cpu_get_gdt(void)1005 cpu_get_gdt(void)
1006 {
1007 return (CPU->cpu_gdt);
1008 }
1009
1010
1011 #if !defined(lwp_getdatamodel)
1012
1013 /*
1014 * Return the datamodel of the given lwp.
1015 */
1016 /*ARGSUSED*/
1017 model_t
lwp_getdatamodel(klwp_t * lwp)1018 lwp_getdatamodel(klwp_t *lwp)
1019 {
1020 return (lwp->lwp_procp->p_model);
1021 }
1022
1023 #endif /* !lwp_getdatamodel */
1024
1025 #if !defined(get_udatamodel)
1026
1027 model_t
get_udatamodel(void)1028 get_udatamodel(void)
1029 {
1030 return (curproc->p_model);
1031 }
1032
1033 #endif /* !get_udatamodel */
1034