xref: /illumos-gate/usr/src/uts/i86pc/os/machdep.c (revision bbf215553c7233fbab8a0afdf1fac74c44781867)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2020 Joyent, Inc.
25  */
26 /*
27  * Copyright (c) 2010, Intel Corporation.
28  * All rights reserved.
29  */
30 
31 #include <sys/types.h>
32 #include <sys/t_lock.h>
33 #include <sys/param.h>
34 #include <sys/segments.h>
35 #include <sys/sysmacros.h>
36 #include <sys/signal.h>
37 #include <sys/systm.h>
38 #include <sys/user.h>
39 #include <sys/mman.h>
40 #include <sys/vm.h>
41 
42 #include <sys/disp.h>
43 #include <sys/class.h>
44 
45 #include <sys/proc.h>
46 #include <sys/buf.h>
47 #include <sys/kmem.h>
48 
49 #include <sys/reboot.h>
50 #include <sys/uadmin.h>
51 #include <sys/callb.h>
52 
53 #include <sys/cred.h>
54 #include <sys/vnode.h>
55 #include <sys/file.h>
56 
57 #include <sys/procfs.h>
58 #include <sys/acct.h>
59 
60 #include <sys/vfs.h>
61 #include <sys/dnlc.h>
62 #include <sys/var.h>
63 #include <sys/cmn_err.h>
64 #include <sys/utsname.h>
65 #include <sys/debug.h>
66 
67 #include <sys/dumphdr.h>
68 #include <sys/bootconf.h>
69 #include <sys/varargs.h>
70 #include <sys/promif.h>
71 #include <sys/modctl.h>
72 
73 #include <sys/consdev.h>
74 #include <sys/frame.h>
75 
76 #include <sys/sunddi.h>
77 #include <sys/ddidmareq.h>
78 #include <sys/psw.h>
79 #include <sys/regset.h>
80 #include <sys/privregs.h>
81 #include <sys/clock.h>
82 #include <sys/tss.h>
83 #include <sys/cpu.h>
84 #include <sys/stack.h>
85 #include <sys/trap.h>
86 #include <sys/pic.h>
87 #include <vm/hat.h>
88 #include <vm/anon.h>
89 #include <vm/as.h>
90 #include <vm/page.h>
91 #include <vm/seg.h>
92 #include <vm/seg_kmem.h>
93 #include <vm/seg_map.h>
94 #include <vm/seg_vn.h>
95 #include <vm/seg_kp.h>
96 #include <vm/hat_i86.h>
97 #include <sys/swap.h>
98 #include <sys/thread.h>
99 #include <sys/sysconf.h>
100 #include <sys/vm_machparam.h>
101 #include <sys/archsystm.h>
102 #include <sys/machsystm.h>
103 #include <sys/machlock.h>
104 #include <sys/x_call.h>
105 #include <sys/instance.h>
106 
107 #include <sys/time.h>
108 #include <sys/smp_impldefs.h>
109 #include <sys/psm_types.h>
110 #include <sys/atomic.h>
111 #include <sys/panic.h>
112 #include <sys/cpuvar.h>
113 #include <sys/dtrace.h>
114 #include <sys/bl.h>
115 #include <sys/nvpair.h>
116 #include <sys/x86_archext.h>
117 #include <sys/pool_pset.h>
118 #include <sys/autoconf.h>
119 #include <sys/mem.h>
120 #include <sys/dumphdr.h>
121 #include <sys/compress.h>
122 #include <sys/cpu_module.h>
123 #if defined(__xpv)
124 #include <sys/hypervisor.h>
125 #include <sys/xpv_panic.h>
126 #endif
127 
128 #include <sys/fastboot.h>
129 #include <sys/machelf.h>
130 #include <sys/kobj.h>
131 #include <sys/multiboot.h>
132 
133 #ifdef	TRAPTRACE
134 #include <sys/traptrace.h>
135 #endif	/* TRAPTRACE */
136 
137 #include <c2/audit.h>
138 #include <sys/clock_impl.h>
139 
140 extern void audit_enterprom(int);
141 extern void audit_exitprom(int);
142 
143 /*
144  * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used.
145  */
146 int	apix_enable = 1;
147 
148 int	apic_nvidia_io_max = 0;	/* no. of NVIDIA i/o apics */
149 
150 /*
151  * Occassionally the kernel knows better whether to power-off or reboot.
152  */
153 int force_shutdown_method = AD_UNKNOWN;
154 
155 /*
156  * The panicbuf array is used to record messages and state:
157  */
158 char panicbuf[PANICBUFSIZE];
159 
160 /*
161  * Flags to control Dynamic Reconfiguration features.
162  */
163 uint64_t plat_dr_options;
164 
165 /*
166  * Maximum physical address for memory DR operations.
167  */
168 uint64_t plat_dr_physmax;
169 
170 /*
171  * maxphys - used during physio
172  * klustsize - used for klustering by swapfs and specfs
173  */
174 int maxphys = 56 * 1024;    /* XXX See vm_subr.c - max b_count in physio */
175 int klustsize = 56 * 1024;
176 
177 caddr_t	p0_va;		/* Virtual address for accessing physical page 0 */
178 
179 /*
180  * defined here, though unused on x86,
181  * to make kstat_fr.c happy.
182  */
183 int vac;
184 
185 void debug_enter(char *);
186 
187 extern void pm_cfb_check_and_powerup(void);
188 extern void pm_cfb_rele(void);
189 
190 extern fastboot_info_t newkernel;
191 
192 /*
193  * Instructions to enable or disable SMAP, respectively.
194  */
195 static const uint8_t clac_instr[3] = { 0x0f, 0x01, 0xca };
196 static const uint8_t stac_instr[3] = { 0x0f, 0x01, 0xcb };
197 
198 /*
199  * Machine dependent code to reboot.
200  * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
201  * to a string to be used as the argument string when rebooting.
202  *
203  * "invoke_cb" is a boolean. It is set to true when mdboot() can safely
204  * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when
205  * we are in a normal shutdown sequence (interrupts are not blocked, the
206  * system is not panic'ing or being suspended).
207  */
208 /*ARGSUSED*/
209 void
mdboot(int cmd,int fcn,char * mdep,boolean_t invoke_cb)210 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb)
211 {
212 	processorid_t bootcpuid = 0;
213 	static int is_first_quiesce = 1;
214 	static int is_first_reset = 1;
215 	int reset_status = 0;
216 	static char fallback_str[] = "Falling back to regular reboot.\n";
217 
218 	if (fcn == AD_FASTREBOOT && !newkernel.fi_valid)
219 		fcn = AD_BOOT;
220 
221 	if (!panicstr) {
222 		kpreempt_disable();
223 		if (fcn == AD_FASTREBOOT) {
224 			mutex_enter(&cpu_lock);
225 			if (CPU_ACTIVE(cpu_get(bootcpuid))) {
226 				affinity_set(bootcpuid);
227 			}
228 			mutex_exit(&cpu_lock);
229 		} else {
230 			affinity_set(CPU_CURRENT);
231 		}
232 	}
233 
234 	if (force_shutdown_method != AD_UNKNOWN)
235 		fcn = force_shutdown_method;
236 
237 	/*
238 	 * XXX - rconsvp is set to NULL to ensure that output messages
239 	 * are sent to the underlying "hardware" device using the
240 	 * monitor's printf routine since we are in the process of
241 	 * either rebooting or halting the machine.
242 	 */
243 	rconsvp = NULL;
244 
245 	/*
246 	 * Print the reboot message now, before pausing other cpus.
247 	 * There is a race condition in the printing support that
248 	 * can deadlock multiprocessor machines.
249 	 */
250 	if (!(fcn == AD_HALT || fcn == AD_POWEROFF))
251 		prom_printf("rebooting...\n");
252 
253 	if (IN_XPV_PANIC())
254 		reset();
255 
256 	/*
257 	 * We can't bring up the console from above lock level, so do it now
258 	 */
259 	pm_cfb_check_and_powerup();
260 
261 	/* make sure there are no more changes to the device tree */
262 	devtree_freeze();
263 
264 	if (invoke_cb)
265 		(void) callb_execute_class(CB_CL_MDBOOT, 0);
266 
267 	/*
268 	 * Clear any unresolved UEs from memory.
269 	 */
270 	page_retire_mdboot();
271 
272 #if defined(__xpv)
273 	/*
274 	 * XXPV	Should probably think some more about how we deal
275 	 *	with panicing before it's really safe to panic.
276 	 *	On hypervisors, we reboot very quickly..  Perhaps panic
277 	 *	should only attempt to recover by rebooting if,
278 	 *	say, we were able to mount the root filesystem,
279 	 *	or if we successfully launched init(8).
280 	 */
281 	if (panicstr && proc_init == NULL)
282 		(void) HYPERVISOR_shutdown(SHUTDOWN_poweroff);
283 #endif
284 	/*
285 	 * stop other cpus and raise our priority.  since there is only
286 	 * one active cpu after this, and our priority will be too high
287 	 * for us to be preempted, we're essentially single threaded
288 	 * from here on out.
289 	 */
290 	(void) spl6();
291 	if (!panicstr) {
292 		mutex_enter(&cpu_lock);
293 		pause_cpus(NULL, NULL);
294 		mutex_exit(&cpu_lock);
295 	}
296 
297 	/*
298 	 * If the system is panicking, the preloaded kernel is valid, and
299 	 * fastreboot_onpanic has been set, and the system has been up for
300 	 * longer than fastreboot_onpanic_uptime (default to 10 minutes),
301 	 * choose Fast Reboot.
302 	 */
303 	if (fcn == AD_BOOT && panicstr && newkernel.fi_valid &&
304 	    fastreboot_onpanic &&
305 	    (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) {
306 		fcn = AD_FASTREBOOT;
307 	}
308 
309 	/*
310 	 * Try to quiesce devices.
311 	 */
312 	if (is_first_quiesce) {
313 		/*
314 		 * Clear is_first_quiesce before calling quiesce_devices()
315 		 * so that if quiesce_devices() causes panics, it will not
316 		 * be invoked again.
317 		 */
318 		is_first_quiesce = 0;
319 
320 		quiesce_active = 1;
321 		quiesce_devices(ddi_root_node(), &reset_status);
322 		if (reset_status == -1) {
323 			if (fcn == AD_FASTREBOOT && !force_fastreboot) {
324 				prom_printf("Driver(s) not capable of fast "
325 				    "reboot.\n");
326 				prom_printf(fallback_str);
327 				fastreboot_capable = 0;
328 				fcn = AD_BOOT;
329 			} else if (fcn != AD_FASTREBOOT)
330 				fastreboot_capable = 0;
331 		}
332 		quiesce_active = 0;
333 	}
334 
335 	/*
336 	 * Try to reset devices. reset_leaves() should only be called
337 	 * a) when there are no other threads that could be accessing devices,
338 	 *    and
339 	 * b) on a system that's not capable of fast reboot (fastreboot_capable
340 	 *    being 0), or on a system where quiesce_devices() failed to
341 	 *    complete (quiesce_active being 1).
342 	 */
343 	if (is_first_reset && (!fastreboot_capable || quiesce_active)) {
344 		/*
345 		 * Clear is_first_reset before calling reset_devices()
346 		 * so that if reset_devices() causes panics, it will not
347 		 * be invoked again.
348 		 */
349 		is_first_reset = 0;
350 		reset_leaves();
351 	}
352 
353 	/* Verify newkernel checksum */
354 	if (fastreboot_capable && fcn == AD_FASTREBOOT &&
355 	    fastboot_cksum_verify(&newkernel) != 0) {
356 		fastreboot_capable = 0;
357 		prom_printf("Fast reboot: checksum failed for the new "
358 		    "kernel.\n");
359 		prom_printf(fallback_str);
360 	}
361 
362 	(void) spl8();
363 
364 	if (fastreboot_capable && fcn == AD_FASTREBOOT) {
365 		/*
366 		 * psm_shutdown is called within fast_reboot()
367 		 */
368 		fast_reboot();
369 	} else {
370 		(*psm_shutdownf)(cmd, fcn);
371 
372 		if (fcn == AD_HALT || fcn == AD_POWEROFF)
373 			halt((char *)NULL);
374 		else
375 			prom_reboot("");
376 	}
377 	/*NOTREACHED*/
378 }
379 
380 /* mdpreboot - may be called prior to mdboot while root fs still mounted */
381 /*ARGSUSED*/
382 void
mdpreboot(int cmd,int fcn,char * mdep)383 mdpreboot(int cmd, int fcn, char *mdep)
384 {
385 	if (fcn == AD_FASTREBOOT && !fastreboot_capable) {
386 		fcn = AD_BOOT;
387 #ifdef	__xpv
388 		cmn_err(CE_WARN, "Fast reboot is not supported on xVM");
389 #else
390 		cmn_err(CE_WARN,
391 		    "Fast reboot is not supported on this platform%s",
392 		    fastreboot_nosup_message());
393 #endif
394 	}
395 
396 	if (fcn == AD_FASTREBOOT) {
397 		fastboot_load_kernel(mdep);
398 		if (!newkernel.fi_valid)
399 			fcn = AD_BOOT;
400 	}
401 
402 	(*psm_preshutdownf)(cmd, fcn);
403 }
404 
405 static void
stop_other_cpus(void)406 stop_other_cpus(void)
407 {
408 	ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */
409 	cpuset_t xcset;
410 
411 	CPUSET_ALL_BUT(xcset, CPU->cpu_id);
412 	xc_priority(0, 0, 0, CPUSET2BV(xcset), mach_cpu_halt);
413 	restore_int_flag(s);
414 }
415 
416 /*
417  *	Machine dependent abort sequence handling
418  */
419 void
abort_sequence_enter(char * msg)420 abort_sequence_enter(char *msg)
421 {
422 	if (abort_enable == 0) {
423 		if (AU_ZONE_AUDITING(GET_KCTX_GZ))
424 			audit_enterprom(0);
425 		return;
426 	}
427 	if (AU_ZONE_AUDITING(GET_KCTX_GZ))
428 		audit_enterprom(1);
429 	debug_enter(msg);
430 	if (AU_ZONE_AUDITING(GET_KCTX_GZ))
431 		audit_exitprom(1);
432 }
433 
434 /*
435  * Enter debugger.  Called when the user types ctrl-alt-d or whenever
436  * code wants to enter the debugger and possibly resume later.
437  *
438  * msg:	message to print, possibly NULL.
439  */
440 void
debug_enter(char * msg)441 debug_enter(char *msg)
442 {
443 	if (dtrace_debugger_init != NULL)
444 		(*dtrace_debugger_init)();
445 
446 	if (msg != NULL || (boothowto & RB_DEBUG))
447 		prom_printf("\n");
448 
449 	if (msg != NULL)
450 		prom_printf("%s\n", msg);
451 
452 	if (boothowto & RB_DEBUG)
453 		kmdb_enter();
454 
455 	if (dtrace_debugger_fini != NULL)
456 		(*dtrace_debugger_fini)();
457 }
458 
459 void
reset(void)460 reset(void)
461 {
462 	extern	void acpi_reset_system();
463 #if !defined(__xpv)
464 	ushort_t *bios_memchk;
465 
466 	/*
467 	 * Can't use psm_map_phys or acpi_reset_system before the hat is
468 	 * initialized.
469 	 */
470 	if (khat_running) {
471 		bios_memchk = (ushort_t *)psm_map_phys(0x472,
472 		    sizeof (ushort_t), PROT_READ | PROT_WRITE);
473 		if (bios_memchk)
474 			*bios_memchk = 0x1234;	/* bios memory check disable */
475 
476 		if (options_dip != NULL &&
477 		    ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0,
478 		    "efi-systab")) {
479 			if (bootops == NULL)
480 				acpi_reset_system();
481 			efi_reset();
482 		}
483 
484 		/*
485 		 * The problem with using stubs is that we can call
486 		 * acpi_reset_system only after the kernel is up and running.
487 		 *
488 		 * We should create a global state to keep track of how far
489 		 * up the kernel is but for the time being we will depend on
490 		 * bootops. bootops cleared in startup_end().
491 		 */
492 		if (bootops == NULL)
493 			acpi_reset_system();
494 	}
495 
496 	pc_reset();
497 #else
498 	if (IN_XPV_PANIC()) {
499 		if (khat_running && bootops == NULL) {
500 			acpi_reset_system();
501 		}
502 
503 		pc_reset();
504 	}
505 
506 	(void) HYPERVISOR_shutdown(SHUTDOWN_reboot);
507 	panic("HYPERVISOR_shutdown() failed");
508 #endif
509 	/*NOTREACHED*/
510 }
511 
512 /*
513  * Halt the machine and return to the monitor
514  */
515 void
halt(char * s)516 halt(char *s)
517 {
518 	stop_other_cpus();	/* send stop signal to other CPUs */
519 	if (s)
520 		prom_printf("(%s) \n", s);
521 	prom_exit_to_mon();
522 	/*NOTREACHED*/
523 }
524 
525 /*
526  * Initiate interrupt redistribution.
527  */
528 void
i_ddi_intr_redist_all_cpus()529 i_ddi_intr_redist_all_cpus()
530 {
531 }
532 
533 /*
534  * XXX These probably ought to live somewhere else
535  * XXX They are called from mem.c
536  */
537 
538 /*
539  * Convert page frame number to an OBMEM page frame number
540  * (i.e. put in the type bits -- zero for this implementation)
541  */
542 pfn_t
impl_obmem_pfnum(pfn_t pf)543 impl_obmem_pfnum(pfn_t pf)
544 {
545 	return (pf);
546 }
547 
548 #ifdef	NM_DEBUG
549 int nmi_test = 0;	/* checked in intentry.s during clock int */
550 int nmtest = -1;
nmfunc1(int arg,struct regs * rp)551 nmfunc1(int arg, struct regs *rp)
552 {
553 	printf("nmi called with arg = %x, regs = %x\n", arg, rp);
554 	nmtest += 50;
555 	if (arg == nmtest) {
556 		printf("ip = %x\n", rp->r_pc);
557 		return (1);
558 	}
559 	return (0);
560 }
561 
562 #endif
563 
564 #include <sys/bootsvcs.h>
565 
566 /* Hacked up initialization for initial kernel check out is HERE. */
567 /* The basic steps are: */
568 /*	kernel bootfuncs definition/initialization for KADB */
569 /*	kadb bootfuncs pointer initialization */
570 /*	putchar/getchar (interrupts disabled) */
571 
572 /* kadb bootfuncs pointer initialization */
573 
574 int
sysp_getchar()575 sysp_getchar()
576 {
577 	int i;
578 	ulong_t s;
579 
580 	if (cons_polledio == NULL) {
581 		/* Uh oh */
582 		prom_printf("getchar called with no console\n");
583 		for (;;)
584 			/* LOOP FOREVER */;
585 	}
586 
587 	s = clear_int_flag();
588 	i = cons_polledio->cons_polledio_getchar(
589 	    cons_polledio->cons_polledio_argument);
590 	restore_int_flag(s);
591 	return (i);
592 }
593 
594 void
sysp_putchar(int c)595 sysp_putchar(int c)
596 {
597 	ulong_t s;
598 
599 	/*
600 	 * We have no alternative but to drop the output on the floor.
601 	 */
602 	if (cons_polledio == NULL ||
603 	    cons_polledio->cons_polledio_putchar == NULL)
604 		return;
605 
606 	s = clear_int_flag();
607 	cons_polledio->cons_polledio_putchar(
608 	    cons_polledio->cons_polledio_argument, c);
609 	restore_int_flag(s);
610 }
611 
612 int
sysp_ischar()613 sysp_ischar()
614 {
615 	int i;
616 	ulong_t s;
617 
618 	if (cons_polledio == NULL ||
619 	    cons_polledio->cons_polledio_ischar == NULL)
620 		return (0);
621 
622 	s = clear_int_flag();
623 	i = cons_polledio->cons_polledio_ischar(
624 	    cons_polledio->cons_polledio_argument);
625 	restore_int_flag(s);
626 	return (i);
627 }
628 
629 int
goany(void)630 goany(void)
631 {
632 	prom_printf("Type any key to continue ");
633 	(void) prom_getchar();
634 	prom_printf("\n");
635 	return (1);
636 }
637 
638 static struct boot_syscalls kern_sysp = {
639 	sysp_getchar,	/*	unchar	(*getchar)();	7  */
640 	sysp_putchar,	/*	int	(*putchar)();	8  */
641 	sysp_ischar,	/*	int	(*ischar)();	9  */
642 };
643 
644 #if defined(__xpv)
645 int using_kern_polledio;
646 #endif
647 
648 void
kadb_uses_kernel()649 kadb_uses_kernel()
650 {
651 	/*
652 	 * This routine is now totally misnamed, since it does not in fact
653 	 * control kadb's I/O; it only controls the kernel's prom_* I/O.
654 	 */
655 	sysp = &kern_sysp;
656 #if defined(__xpv)
657 	using_kern_polledio = 1;
658 #endif
659 }
660 
661 /*
662  *	the interface to the outside world
663  */
664 
665 /*
666  * poll_port -- wait for a register to achieve a
667  *		specific state.  Arguments are a mask of bits we care about,
668  *		and two sub-masks.  To return normally, all the bits in the
669  *		first sub-mask must be ON, all the bits in the second sub-
670  *		mask must be OFF.  If about seconds pass without the register
671  *		achieving the desired bit configuration, we return 1, else
672  *		0.
673  */
674 int
poll_port(ushort_t port,ushort_t mask,ushort_t onbits,ushort_t offbits)675 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits)
676 {
677 	int i;
678 	ushort_t maskval;
679 
680 	for (i = 500000; i; i--) {
681 		maskval = inb(port) & mask;
682 		if (((maskval & onbits) == onbits) &&
683 		    ((maskval & offbits) == 0))
684 			return (0);
685 		drv_usecwait(10);
686 	}
687 	return (1);
688 }
689 
690 /*
691  * set_idle_cpu is called from idle() when a CPU becomes idle.
692  */
693 /*LINTED: static unused */
694 static uint_t last_idle_cpu;
695 
696 /*ARGSUSED*/
697 void
set_idle_cpu(int cpun)698 set_idle_cpu(int cpun)
699 {
700 	last_idle_cpu = cpun;
701 	(*psm_set_idle_cpuf)(cpun);
702 }
703 
704 /*
705  * unset_idle_cpu is called from idle() when a CPU is no longer idle.
706  */
707 /*ARGSUSED*/
708 void
unset_idle_cpu(int cpun)709 unset_idle_cpu(int cpun)
710 {
711 	(*psm_unset_idle_cpuf)(cpun);
712 }
713 
714 /*
715  * This routine is almost correct now, but not quite.  It still needs the
716  * equivalent concept of "hres_last_tick", just like on the sparc side.
717  * The idea is to take a snapshot of the hi-res timer while doing the
718  * hrestime_adj updates under hres_lock in locore, so that the small
719  * interval between interrupt assertion and interrupt processing is
720  * accounted for correctly.  Once we have this, the code below should
721  * be modified to subtract off hres_last_tick rather than hrtime_base.
722  *
723  * I'd have done this myself, but I don't have source to all of the
724  * vendor-specific hi-res timer routines (grrr...).  The generic hook I
725  * need is something like "gethrtime_unlocked()", which would be just like
726  * gethrtime() but would assume that you're already holding CLOCK_LOCK().
727  * This is what the GET_HRTIME() macro is for on sparc (although it also
728  * serves the function of making time available without a function call
729  * so you don't take a register window overflow while traps are disabled).
730  */
731 void
pc_gethrestime(timestruc_t * tp)732 pc_gethrestime(timestruc_t *tp)
733 {
734 	int lock_prev;
735 	timestruc_t now;
736 	int nslt;		/* nsec since last tick */
737 	int adj;		/* amount of adjustment to apply */
738 
739 loop:
740 	lock_prev = hres_lock;
741 	now = hrestime;
742 	nslt = (int)(gethrtime() - hres_last_tick);
743 	if (nslt < 0) {
744 		/*
745 		 * nslt < 0 means a tick came between sampling
746 		 * gethrtime() and hres_last_tick; restart the loop
747 		 */
748 
749 		goto loop;
750 	}
751 	now.tv_nsec += nslt;
752 	if (hrestime_adj != 0) {
753 		if (hrestime_adj > 0) {
754 			adj = (nslt >> ADJ_SHIFT);
755 			if (adj > hrestime_adj)
756 				adj = (int)hrestime_adj;
757 		} else {
758 			adj = -(nslt >> ADJ_SHIFT);
759 			if (adj < hrestime_adj)
760 				adj = (int)hrestime_adj;
761 		}
762 		now.tv_nsec += adj;
763 	}
764 	while ((unsigned long)now.tv_nsec >= NANOSEC) {
765 
766 		/*
767 		 * We might have a large adjustment or have been in the
768 		 * debugger for a long time; take care of (at most) four
769 		 * of those missed seconds (tv_nsec is 32 bits, so
770 		 * anything >4s will be wrapping around).  However,
771 		 * anything more than 2 seconds out of sync will trigger
772 		 * timedelta from clock() to go correct the time anyway,
773 		 * so do what we can, and let the big crowbar do the
774 		 * rest.  A similar correction while loop exists inside
775 		 * hres_tick(); in all cases we'd like tv_nsec to
776 		 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing
777 		 * user processes, but if tv_sec's a little behind for a
778 		 * little while, that's OK; time still monotonically
779 		 * increases.
780 		 */
781 
782 		now.tv_nsec -= NANOSEC;
783 		now.tv_sec++;
784 	}
785 	if ((hres_lock & ~1) != lock_prev)
786 		goto loop;
787 
788 	*tp = now;
789 }
790 
791 void
gethrestime_lasttick(timespec_t * tp)792 gethrestime_lasttick(timespec_t *tp)
793 {
794 	int s;
795 
796 	s = hr_clock_lock();
797 	*tp = hrestime;
798 	hr_clock_unlock(s);
799 }
800 
801 time_t
gethrestime_sec(void)802 gethrestime_sec(void)
803 {
804 	timestruc_t now;
805 
806 	gethrestime(&now);
807 	return (now.tv_sec);
808 }
809 
810 /*
811  * Initialize a kernel thread's stack
812  */
813 
814 caddr_t
thread_stk_init(caddr_t stk)815 thread_stk_init(caddr_t stk)
816 {
817 	ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0);
818 	return (stk - SA(MINFRAME));
819 }
820 
821 /*
822  * Initialize lwp's kernel stack.
823  */
824 
825 #ifdef TRAPTRACE
826 /*
827  * There's a tricky interdependency here between use of sysenter and
828  * TRAPTRACE which needs recording to avoid future confusion (this is
829  * about the third time I've re-figured this out ..)
830  *
831  * Here's how debugging lcall works with TRAPTRACE.
832  *
833  * 1 We're in userland with a breakpoint on the lcall instruction.
834  * 2 We execute the instruction - the instruction pushes the userland
835  *   %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel
836  *   via the call gate.
837  * 3 The hardware raises a debug trap in kernel mode, the hardware
838  *   pushes %efl, %cs, %eip and gets to dbgtrap via the idt.
839  * 4 dbgtrap pushes the error code and trapno and calls cmntrap
840  * 5 cmntrap finishes building a trap frame
841  * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk
842  *   off the stack into the traptrace buffer.
843  *
844  * This means that the traptrace buffer contains the wrong values in
845  * %esp and %ss, but everything else in there is correct.
846  *
847  * Here's how debugging sysenter works with TRAPTRACE.
848  *
849  * a We're in userland with a breakpoint on the sysenter instruction.
850  * b We execute the instruction - the instruction pushes -nothing-
851  *   on the stack, but sets %cs, %eip, %ss, %esp to prearranged
852  *   values to take us to sys_sysenter, at the top of the lwp's
853  *   stack.
854  * c goto 3
855  *
856  * At this point, because we got into the kernel without the requisite
857  * five pushes on the stack, if we didn't make extra room, we'd
858  * end up with the TRACE_REGS macro fetching the saved %ss and %esp
859  * values from negative (unmapped) stack addresses -- which really bites.
860  * That's why we do the '-= 8' below.
861  *
862  * XXX	Note that reading "up" lwp0's stack works because t0 is declared
863  *	right next to t0stack in locore.s
864  */
865 #endif
866 
867 caddr_t
lwp_stk_init(klwp_t * lwp,caddr_t stk)868 lwp_stk_init(klwp_t *lwp, caddr_t stk)
869 {
870 	caddr_t oldstk;
871 	struct pcb *pcb = &lwp->lwp_pcb;
872 
873 	oldstk = stk;
874 	stk -= SA(sizeof (struct regs) + SA(MINFRAME));
875 #ifdef TRAPTRACE
876 	stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */
877 #endif
878 	stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul));
879 	bzero(stk, oldstk - stk);
880 	lwp->lwp_regs = (void *)(stk + SA(MINFRAME));
881 
882 	/*
883 	 * Arrange that the virtualized %fs and %gs GDT descriptors
884 	 * have a well-defined initial state (present, ring 3
885 	 * and of type data).
886 	 */
887 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE)
888 		pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
889 	else
890 		pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
891 	lwp_installctx(lwp);
892 	return (stk);
893 }
894 
895 /*
896  * Use this opportunity to free any dynamically allocated fp storage.
897  */
898 void
lwp_stk_fini(klwp_t * lwp)899 lwp_stk_fini(klwp_t *lwp)
900 {
901 	fp_lwp_cleanup(lwp);
902 }
903 
904 void
lwp_fp_init(klwp_t * lwp)905 lwp_fp_init(klwp_t *lwp)
906 {
907 	fp_lwp_init(lwp);
908 }
909 
910 /*
911  * If we're not the panic CPU, we wait in panic_idle for reboot.
912  */
913 void
panic_idle(void)914 panic_idle(void)
915 {
916 	splx(ipltospl(CLOCK_LEVEL));
917 	(void) setjmp(&curthread->t_pcb);
918 
919 	dumpsys_helper();
920 
921 #ifndef __xpv
922 	for (;;)
923 		i86_halt();
924 #else
925 	for (;;)
926 		;
927 #endif
928 }
929 
930 /*
931  * Stop the other CPUs by cross-calling them and forcing them to enter
932  * the panic_idle() loop above.
933  */
934 /*ARGSUSED*/
935 void
panic_stopcpus(cpu_t * cp,kthread_t * t,int spl)936 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl)
937 {
938 	processorid_t i;
939 	cpuset_t xcset;
940 
941 	/*
942 	 * In the case of a Xen panic, the hypervisor has already stopped
943 	 * all of the CPUs.
944 	 */
945 	if (!IN_XPV_PANIC()) {
946 		(void) splzs();
947 
948 		CPUSET_ALL_BUT(xcset, cp->cpu_id);
949 		xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle);
950 	}
951 
952 	for (i = 0; i < NCPU; i++) {
953 		if (i != cp->cpu_id && cpu[i] != NULL &&
954 		    (cpu[i]->cpu_flags & CPU_EXISTS))
955 			cpu[i]->cpu_flags |= CPU_QUIESCED;
956 	}
957 }
958 
959 /*
960  * Platform callback following each entry to panicsys().
961  */
962 /*ARGSUSED*/
963 void
panic_enter_hw(int spl)964 panic_enter_hw(int spl)
965 {
966 	/* Nothing to do here */
967 }
968 
969 /*
970  * Platform-specific code to execute after panicstr is set: we invoke
971  * the PSM entry point to indicate that a panic has occurred.
972  */
973 /*ARGSUSED*/
974 void
panic_quiesce_hw(panic_data_t * pdp)975 panic_quiesce_hw(panic_data_t *pdp)
976 {
977 	psm_notifyf(PSM_PANIC_ENTER);
978 
979 	cmi_panic_callback();
980 
981 #ifdef	TRAPTRACE
982 	/*
983 	 * Turn off TRAPTRACE
984 	 */
985 	TRAPTRACE_FREEZE;
986 #endif	/* TRAPTRACE */
987 }
988 
989 /*
990  * Platform callback prior to writing crash dump.
991  */
992 /*ARGSUSED*/
993 void
panic_dump_hw(int spl)994 panic_dump_hw(int spl)
995 {
996 	/* Nothing to do here */
997 }
998 
999 void *
plat_traceback(void * fpreg)1000 plat_traceback(void *fpreg)
1001 {
1002 #ifdef __xpv
1003 	if (IN_XPV_PANIC())
1004 		return (xpv_traceback(fpreg));
1005 #endif
1006 	return (fpreg);
1007 }
1008 
1009 /*ARGSUSED*/
1010 void
plat_tod_fault(enum tod_fault_type tod_bad)1011 plat_tod_fault(enum tod_fault_type tod_bad)
1012 {}
1013 
1014 /*ARGSUSED*/
1015 int
blacklist(int cmd,const char * scheme,nvlist_t * fmri,const char * class)1016 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class)
1017 {
1018 	return (ENOTSUP);
1019 }
1020 
1021 /*
1022  * The underlying console output routines are protected by raising IPL in case
1023  * we are still calling into the early boot services.  Once we start calling
1024  * the kernel console emulator, it will disable interrupts completely during
1025  * character rendering (see sysp_putchar, for example).  Refer to the comments
1026  * and code in common/os/console.c for more information on these callbacks.
1027  */
1028 /*ARGSUSED*/
1029 int
console_enter(int busy)1030 console_enter(int busy)
1031 {
1032 	return (splzs());
1033 }
1034 
1035 /*ARGSUSED*/
1036 void
console_exit(int busy,int spl)1037 console_exit(int busy, int spl)
1038 {
1039 	splx(spl);
1040 }
1041 
1042 /*
1043  * Allocate a region of virtual address space, unmapped.
1044  * Stubbed out except on sparc, at least for now.
1045  */
1046 /*ARGSUSED*/
1047 void *
boot_virt_alloc(void * addr,size_t size)1048 boot_virt_alloc(void *addr, size_t size)
1049 {
1050 	return (addr);
1051 }
1052 
1053 void
tenmicrosec(void)1054 tenmicrosec(void)
1055 {
1056 	extern int gethrtime_hires;
1057 
1058 	if (gethrtime_hires) {
1059 		hrtime_t start, end;
1060 		start = end =  gethrtime();
1061 		while ((end - start) < (10 * (NANOSEC / MICROSEC))) {
1062 			SMT_PAUSE();
1063 			end = gethrtime();
1064 		}
1065 	} else {
1066 #if defined(__xpv)
1067 		hrtime_t newtime;
1068 
1069 		newtime = xpv_gethrtime() + 10000; /* now + 10 us */
1070 		while (xpv_gethrtime() < newtime)
1071 			SMT_PAUSE();
1072 #else	/* __xpv */
1073 		panic("TSC was not calibrated!");
1074 #endif	/* __xpv */
1075 	}
1076 }
1077 
1078 /*
1079  * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
1080  * long, and it fills in the array with the time spent on cpu in
1081  * each of the mstates, where time is returned in nsec.
1082  *
1083  * No guarantee is made that the returned values in times[] will
1084  * monotonically increase on sequential calls, although this will
1085  * be true in the long run. Any such guarantee must be handled by
1086  * the caller, if needed. This can happen if we fail to account
1087  * for elapsed time due to a generation counter conflict, yet we
1088  * did account for it on a prior call (see below).
1089  *
1090  * The complication is that the cpu in question may be updating
1091  * its microstate at the same time that we are reading it.
1092  * Because the microstate is only updated when the CPU's state
1093  * changes, the values in cpu_intracct[] can be indefinitely out
1094  * of date. To determine true current values, it is necessary to
1095  * compare the current time with cpu_mstate_start, and add the
1096  * difference to times[cpu_mstate].
1097  *
1098  * This can be a problem if those values are changing out from
1099  * under us. Because the code path in new_cpu_mstate() is
1100  * performance critical, we have not added a lock to it. Instead,
1101  * we have added a generation counter. Before beginning
1102  * modifications, the counter is set to 0. After modifications,
1103  * it is set to the old value plus one.
1104  *
1105  * get_cpu_mstate() will not consider the values of cpu_mstate
1106  * and cpu_mstate_start to be usable unless the value of
1107  * cpu_mstate_gen is both non-zero and unchanged, both before and
1108  * after reading the mstate information. Note that we must
1109  * protect against out-of-order loads around accesses to the
1110  * generation counter. Also, this is a best effort approach in
1111  * that we do not retry should the counter be found to have
1112  * changed.
1113  *
1114  * cpu_intracct[] is used to identify time spent in each CPU
1115  * mstate while handling interrupts. Such time should be reported
1116  * against system time, and so is subtracted out from its
1117  * corresponding cpu_acct[] time and added to
1118  * cpu_acct[CMS_SYSTEM].
1119  */
1120 
1121 void
get_cpu_mstate(cpu_t * cpu,hrtime_t * times)1122 get_cpu_mstate(cpu_t *cpu, hrtime_t *times)
1123 {
1124 	int i;
1125 	hrtime_t now, start;
1126 	uint16_t gen;
1127 	uint16_t state;
1128 	hrtime_t intracct[NCMSTATES];
1129 
1130 	/*
1131 	 * Load all volatile state under the protection of membar.
1132 	 * cpu_acct[cpu_mstate] must be loaded to avoid double counting
1133 	 * of (now - cpu_mstate_start) by a change in CPU mstate that
1134 	 * arrives after we make our last check of cpu_mstate_gen.
1135 	 */
1136 
1137 	now = gethrtime_unscaled();
1138 	gen = cpu->cpu_mstate_gen;
1139 
1140 	membar_consumer();	/* guarantee load ordering */
1141 	start = cpu->cpu_mstate_start;
1142 	state = cpu->cpu_mstate;
1143 	for (i = 0; i < NCMSTATES; i++) {
1144 		intracct[i] = cpu->cpu_intracct[i];
1145 		times[i] = cpu->cpu_acct[i];
1146 	}
1147 	membar_consumer();	/* guarantee load ordering */
1148 
1149 	if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start)
1150 		times[state] += now - start;
1151 
1152 	for (i = 0; i < NCMSTATES; i++) {
1153 		if (i == CMS_SYSTEM)
1154 			continue;
1155 		times[i] -= intracct[i];
1156 		if (times[i] < 0) {
1157 			intracct[i] += times[i];
1158 			times[i] = 0;
1159 		}
1160 		times[CMS_SYSTEM] += intracct[i];
1161 		scalehrtime(&times[i]);
1162 	}
1163 	scalehrtime(&times[CMS_SYSTEM]);
1164 }
1165 
1166 /*
1167  * This is a version of the rdmsr instruction that allows
1168  * an error code to be returned in the case of failure.
1169  */
1170 int
checked_rdmsr(uint_t msr,uint64_t * value)1171 checked_rdmsr(uint_t msr, uint64_t *value)
1172 {
1173 	if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1174 		return (ENOTSUP);
1175 	*value = rdmsr(msr);
1176 	return (0);
1177 }
1178 
1179 /*
1180  * This is a version of the wrmsr instruction that allows
1181  * an error code to be returned in the case of failure.
1182  */
1183 int
checked_wrmsr(uint_t msr,uint64_t value)1184 checked_wrmsr(uint_t msr, uint64_t value)
1185 {
1186 	if (!is_x86_feature(x86_featureset, X86FSET_MSR))
1187 		return (ENOTSUP);
1188 	wrmsr(msr, value);
1189 	return (0);
1190 }
1191 
1192 /*
1193  * The mem driver's usual method of using hat_devload() to establish a
1194  * temporary mapping will not work for foreign pages mapped into this
1195  * domain or for the special hypervisor-provided pages.  For the foreign
1196  * pages, we often don't know which domain owns them, so we can't ask the
1197  * hypervisor to set up a new mapping.  For the other pages, we don't have
1198  * a pfn, so we can't create a new PTE.  For these special cases, we do a
1199  * direct uiomove() from the existing kernel virtual address.
1200  */
1201 /*ARGSUSED*/
1202 int
plat_mem_do_mmio(struct uio * uio,enum uio_rw rw)1203 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw)
1204 {
1205 #if defined(__xpv)
1206 	void *va = (void *)(uintptr_t)uio->uio_loffset;
1207 	off_t pageoff = uio->uio_loffset & PAGEOFFSET;
1208 	size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
1209 	    (size_t)uio->uio_iov->iov_len);
1210 
1211 	if ((rw == UIO_READ &&
1212 	    (va == HYPERVISOR_shared_info || va == xen_info)) ||
1213 	    (pfn_is_foreign(hat_getpfnum(kas.a_hat, va))))
1214 		return (uiomove(va, nbytes, rw, uio));
1215 #endif
1216 	return (ENOTSUP);
1217 }
1218 
1219 pgcnt_t
num_phys_pages()1220 num_phys_pages()
1221 {
1222 	pgcnt_t npages = 0;
1223 	struct memlist *mp;
1224 
1225 #if defined(__xpv)
1226 	if (DOMAIN_IS_INITDOMAIN(xen_info))
1227 		return (xpv_nr_phys_pages());
1228 #endif /* __xpv */
1229 
1230 	for (mp = phys_install; mp != NULL; mp = mp->ml_next)
1231 		npages += mp->ml_size >> PAGESHIFT;
1232 
1233 	return (npages);
1234 }
1235 
1236 /* cpu threshold for compressed dumps */
1237 #ifdef _LP64
1238 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_64_MINCPU;
1239 #else
1240 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_32_MINCPU;
1241 #endif
1242 
1243 int
dump_plat_addr()1244 dump_plat_addr()
1245 {
1246 #ifdef __xpv
1247 	pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1248 	mem_vtop_t mem_vtop;
1249 	int cnt;
1250 
1251 	/*
1252 	 * On the hypervisor, we want to dump the page with shared_info on it.
1253 	 */
1254 	if (!IN_XPV_PANIC()) {
1255 		mem_vtop.m_as = &kas;
1256 		mem_vtop.m_va = HYPERVISOR_shared_info;
1257 		mem_vtop.m_pfn = pfn;
1258 		dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1259 		cnt = 1;
1260 	} else {
1261 		cnt = dump_xpv_addr();
1262 	}
1263 	return (cnt);
1264 #else
1265 	return (0);
1266 #endif
1267 }
1268 
1269 void
dump_plat_pfn()1270 dump_plat_pfn()
1271 {
1272 #ifdef __xpv
1273 	pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1274 
1275 	if (!IN_XPV_PANIC())
1276 		dumpvp_write(&pfn, sizeof (pfn));
1277 	else
1278 		dump_xpv_pfn();
1279 #endif
1280 }
1281 
1282 /*ARGSUSED*/
1283 int
dump_plat_data(void * dump_cbuf)1284 dump_plat_data(void *dump_cbuf)
1285 {
1286 #ifdef __xpv
1287 	uint32_t csize;
1288 	int cnt;
1289 
1290 	if (!IN_XPV_PANIC()) {
1291 		csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf,
1292 		    PAGESIZE);
1293 		dumpvp_write(&csize, sizeof (uint32_t));
1294 		dumpvp_write(dump_cbuf, csize);
1295 		cnt = 1;
1296 	} else {
1297 		cnt = dump_xpv_data(dump_cbuf);
1298 	}
1299 	return (cnt);
1300 #else
1301 	return (0);
1302 #endif
1303 }
1304 
1305 /*
1306  * Calculates a linear address, given the CS selector and PC values
1307  * by looking up the %cs selector process's LDT or the CPU's GDT.
1308  * proc->p_ldtlock must be held across this call.
1309  */
1310 int
linear_pc(struct regs * rp,proc_t * p,caddr_t * linearp)1311 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1312 {
1313 	user_desc_t	*descrp;
1314 	caddr_t		baseaddr;
1315 	uint16_t	idx = SELTOIDX(rp->r_cs);
1316 
1317 	ASSERT(rp->r_cs <= 0xFFFF);
1318 	ASSERT(MUTEX_HELD(&p->p_ldtlock));
1319 
1320 	if (SELISLDT(rp->r_cs)) {
1321 		/*
1322 		 * Currently 64 bit processes cannot have private LDTs.
1323 		 */
1324 		ASSERT(p->p_model != DATAMODEL_LP64);
1325 
1326 		if (p->p_ldt == NULL)
1327 			return (-1);
1328 
1329 		descrp = &p->p_ldt[idx];
1330 		baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1331 
1332 		/*
1333 		 * Calculate the linear address (wraparound is not only ok,
1334 		 * it's expected behavior).  The cast to uint32_t is because
1335 		 * LDT selectors are only allowed in 32-bit processes.
1336 		 */
1337 		*linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1338 		    rp->r_pc);
1339 	} else {
1340 #ifdef DEBUG
1341 		descrp = &CPU->cpu_gdt[idx];
1342 		baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1343 		/* GDT-based descriptors' base addresses should always be 0 */
1344 		ASSERT(baseaddr == 0);
1345 #endif
1346 		*linearp = (caddr_t)(uintptr_t)rp->r_pc;
1347 	}
1348 
1349 	return (0);
1350 }
1351 
1352 /*
1353  * The implementation of dtrace_linear_pc is similar to the that of
1354  * linear_pc, above, but here we acquire p_ldtlock before accessing
1355  * p_ldt.  This implementation is used by the pid provider; we prefix
1356  * it with "dtrace_" to avoid inducing spurious tracing events.
1357  */
1358 int
dtrace_linear_pc(struct regs * rp,proc_t * p,caddr_t * linearp)1359 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1360 {
1361 	user_desc_t	*descrp;
1362 	caddr_t		baseaddr;
1363 	uint16_t	idx = SELTOIDX(rp->r_cs);
1364 
1365 	ASSERT(rp->r_cs <= 0xFFFF);
1366 
1367 	if (SELISLDT(rp->r_cs)) {
1368 		/*
1369 		 * Currently 64 bit processes cannot have private LDTs.
1370 		 */
1371 		ASSERT(p->p_model != DATAMODEL_LP64);
1372 
1373 		mutex_enter(&p->p_ldtlock);
1374 		if (p->p_ldt == NULL) {
1375 			mutex_exit(&p->p_ldtlock);
1376 			return (-1);
1377 		}
1378 		descrp = &p->p_ldt[idx];
1379 		baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1380 		mutex_exit(&p->p_ldtlock);
1381 
1382 		/*
1383 		 * Calculate the linear address (wraparound is not only ok,
1384 		 * it's expected behavior).  The cast to uint32_t is because
1385 		 * LDT selectors are only allowed in 32-bit processes.
1386 		 */
1387 		*linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1388 		    rp->r_pc);
1389 	} else {
1390 #ifdef DEBUG
1391 		descrp = &CPU->cpu_gdt[idx];
1392 		baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1393 		/* GDT-based descriptors' base addresses should always be 0 */
1394 		ASSERT(baseaddr == 0);
1395 #endif
1396 		*linearp = (caddr_t)(uintptr_t)rp->r_pc;
1397 	}
1398 
1399 	return (0);
1400 }
1401 
1402 /*
1403  * We need to post a soft interrupt to reprogram the lbolt cyclic when
1404  * switching from event to cyclic driven lbolt. The following code adds
1405  * and posts the softint for x86.
1406  */
1407 static ddi_softint_hdl_impl_t lbolt_softint_hdl =
1408 	{0, 0, NULL, NULL, 0, NULL, NULL, NULL};
1409 
1410 void
lbolt_softint_add(void)1411 lbolt_softint_add(void)
1412 {
1413 	(void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL,
1414 	    (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL);
1415 }
1416 
1417 void
lbolt_softint_post(void)1418 lbolt_softint_post(void)
1419 {
1420 	(*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending);
1421 }
1422 
1423 boolean_t
plat_dr_check_capability(uint64_t features)1424 plat_dr_check_capability(uint64_t features)
1425 {
1426 	return ((plat_dr_options & features) == features);
1427 }
1428 
1429 boolean_t
plat_dr_support_cpu(void)1430 plat_dr_support_cpu(void)
1431 {
1432 	return (plat_dr_options & PLAT_DR_FEATURE_CPU);
1433 }
1434 
1435 boolean_t
plat_dr_support_memory(void)1436 plat_dr_support_memory(void)
1437 {
1438 	return (plat_dr_options & PLAT_DR_FEATURE_MEMORY);
1439 }
1440 
1441 void
plat_dr_enable_capability(uint64_t features)1442 plat_dr_enable_capability(uint64_t features)
1443 {
1444 	atomic_or_64(&plat_dr_options, features);
1445 }
1446 
1447 void
plat_dr_disable_capability(uint64_t features)1448 plat_dr_disable_capability(uint64_t features)
1449 {
1450 	atomic_and_64(&plat_dr_options, ~features);
1451 }
1452 
1453 /*
1454  * If SMAP is supported, look through hi_calls and inline
1455  * calls to smap_enable() to clac and smap_disable() to stac.
1456  */
1457 void
hotinline_smap(hotinline_desc_t * hid)1458 hotinline_smap(hotinline_desc_t *hid)
1459 {
1460 	if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE)
1461 		return;
1462 
1463 	if (strcmp(hid->hid_symname, "smap_enable") == 0) {
1464 		bcopy(clac_instr, (void *)hid->hid_instr_offset,
1465 		    sizeof (clac_instr));
1466 	} else if (strcmp(hid->hid_symname, "smap_disable") == 0) {
1467 		bcopy(stac_instr, (void *)hid->hid_instr_offset,
1468 		    sizeof (stac_instr));
1469 	}
1470 }
1471 
1472 /*
1473  * Loop through hi_calls and hand off the inlining to
1474  * the appropriate calls.
1475  */
1476 void
do_hotinlines(struct module * mp)1477 do_hotinlines(struct module *mp)
1478 {
1479 	for (hotinline_desc_t *hid = mp->hi_calls; hid != NULL;
1480 	    hid = hid->hid_next) {
1481 #if !defined(__xpv)
1482 		hotinline_smap(hid);
1483 #endif	/* __xpv */
1484 	}
1485 }
1486