xref: /illumos-gate/usr/src/uts/i86pc/os/machdep.c (revision 46e5ca4c180bbc8cb48be79bc045e873add461ac)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include <sys/types.h>
28 #include <sys/t_lock.h>
29 #include <sys/param.h>
30 #include <sys/segments.h>
31 #include <sys/sysmacros.h>
32 #include <sys/signal.h>
33 #include <sys/systm.h>
34 #include <sys/user.h>
35 #include <sys/mman.h>
36 #include <sys/vm.h>
37 
38 #include <sys/disp.h>
39 #include <sys/class.h>
40 
41 #include <sys/proc.h>
42 #include <sys/buf.h>
43 #include <sys/kmem.h>
44 
45 #include <sys/reboot.h>
46 #include <sys/uadmin.h>
47 #include <sys/callb.h>
48 
49 #include <sys/cred.h>
50 #include <sys/vnode.h>
51 #include <sys/file.h>
52 
53 #include <sys/procfs.h>
54 #include <sys/acct.h>
55 
56 #include <sys/vfs.h>
57 #include <sys/dnlc.h>
58 #include <sys/var.h>
59 #include <sys/cmn_err.h>
60 #include <sys/utsname.h>
61 #include <sys/debug.h>
62 
63 #include <sys/dumphdr.h>
64 #include <sys/bootconf.h>
65 #include <sys/varargs.h>
66 #include <sys/promif.h>
67 #include <sys/modctl.h>
68 
69 #include <sys/consdev.h>
70 #include <sys/frame.h>
71 
72 #include <sys/sunddi.h>
73 #include <sys/ddidmareq.h>
74 #include <sys/psw.h>
75 #include <sys/regset.h>
76 #include <sys/privregs.h>
77 #include <sys/clock.h>
78 #include <sys/tss.h>
79 #include <sys/cpu.h>
80 #include <sys/stack.h>
81 #include <sys/trap.h>
82 #include <sys/pic.h>
83 #include <vm/hat.h>
84 #include <vm/anon.h>
85 #include <vm/as.h>
86 #include <vm/page.h>
87 #include <vm/seg.h>
88 #include <vm/seg_kmem.h>
89 #include <vm/seg_map.h>
90 #include <vm/seg_vn.h>
91 #include <vm/seg_kp.h>
92 #include <vm/hat_i86.h>
93 #include <sys/swap.h>
94 #include <sys/thread.h>
95 #include <sys/sysconf.h>
96 #include <sys/vm_machparam.h>
97 #include <sys/archsystm.h>
98 #include <sys/machsystm.h>
99 #include <sys/machlock.h>
100 #include <sys/x_call.h>
101 #include <sys/instance.h>
102 
103 #include <sys/time.h>
104 #include <sys/smp_impldefs.h>
105 #include <sys/psm_types.h>
106 #include <sys/atomic.h>
107 #include <sys/panic.h>
108 #include <sys/cpuvar.h>
109 #include <sys/dtrace.h>
110 #include <sys/bl.h>
111 #include <sys/nvpair.h>
112 #include <sys/x86_archext.h>
113 #include <sys/pool_pset.h>
114 #include <sys/autoconf.h>
115 #include <sys/mem.h>
116 #include <sys/dumphdr.h>
117 #include <sys/compress.h>
118 #include <sys/cpu_module.h>
119 #if defined(__xpv)
120 #include <sys/hypervisor.h>
121 #include <sys/xpv_panic.h>
122 #endif
123 
124 #include <sys/fastboot.h>
125 #include <sys/machelf.h>
126 #include <sys/kobj.h>
127 #include <sys/multiboot.h>
128 
129 #ifdef	TRAPTRACE
130 #include <sys/traptrace.h>
131 #endif	/* TRAPTRACE */
132 
133 extern void audit_enterprom(int);
134 extern void audit_exitprom(int);
135 
136 /*
137  * Occassionally the kernel knows better whether to power-off or reboot.
138  */
139 int force_shutdown_method = AD_UNKNOWN;
140 
141 /*
142  * The panicbuf array is used to record messages and state:
143  */
144 char panicbuf[PANICBUFSIZE];
145 
146 /*
147  * maxphys - used during physio
148  * klustsize - used for klustering by swapfs and specfs
149  */
150 int maxphys = 56 * 1024;    /* XXX See vm_subr.c - max b_count in physio */
151 int klustsize = 56 * 1024;
152 
153 caddr_t	p0_va;		/* Virtual address for accessing physical page 0 */
154 
155 /*
156  * defined here, though unused on x86,
157  * to make kstat_fr.c happy.
158  */
159 int vac;
160 
161 void debug_enter(char *);
162 
163 extern void pm_cfb_check_and_powerup(void);
164 extern void pm_cfb_rele(void);
165 
166 extern fastboot_info_t newkernel;
167 
168 /*
169  * Machine dependent code to reboot.
170  * "mdep" is interpreted as a character pointer; if non-null, it is a pointer
171  * to a string to be used as the argument string when rebooting.
172  *
173  * "invoke_cb" is a boolean. It is set to true when mdboot() can safely
174  * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when
175  * we are in a normal shutdown sequence (interrupts are not blocked, the
176  * system is not panic'ing or being suspended).
177  */
178 /*ARGSUSED*/
179 void
180 mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb)
181 {
182 	processorid_t bootcpuid = 0;
183 	static int is_first_quiesce = 1;
184 	static int is_first_reset = 1;
185 	int reset_status = 0;
186 	static char fallback_str[] = "Falling back to regular reboot.\n";
187 
188 	if (fcn == AD_FASTREBOOT && !newkernel.fi_valid)
189 		fcn = AD_BOOT;
190 
191 	if (!panicstr) {
192 		kpreempt_disable();
193 		if (fcn == AD_FASTREBOOT) {
194 			mutex_enter(&cpu_lock);
195 			if (CPU_ACTIVE(cpu_get(bootcpuid))) {
196 				affinity_set(bootcpuid);
197 			}
198 			mutex_exit(&cpu_lock);
199 		} else {
200 			affinity_set(CPU_CURRENT);
201 		}
202 	}
203 
204 	if (force_shutdown_method != AD_UNKNOWN)
205 		fcn = force_shutdown_method;
206 
207 	/*
208 	 * XXX - rconsvp is set to NULL to ensure that output messages
209 	 * are sent to the underlying "hardware" device using the
210 	 * monitor's printf routine since we are in the process of
211 	 * either rebooting or halting the machine.
212 	 */
213 	rconsvp = NULL;
214 
215 	/*
216 	 * Print the reboot message now, before pausing other cpus.
217 	 * There is a race condition in the printing support that
218 	 * can deadlock multiprocessor machines.
219 	 */
220 	if (!(fcn == AD_HALT || fcn == AD_POWEROFF))
221 		prom_printf("rebooting...\n");
222 
223 	if (IN_XPV_PANIC())
224 		reset();
225 
226 	/*
227 	 * We can't bring up the console from above lock level, so do it now
228 	 */
229 	pm_cfb_check_and_powerup();
230 
231 	/* make sure there are no more changes to the device tree */
232 	devtree_freeze();
233 
234 	if (invoke_cb)
235 		(void) callb_execute_class(CB_CL_MDBOOT, NULL);
236 
237 	/*
238 	 * Clear any unresolved UEs from memory.
239 	 */
240 	page_retire_mdboot();
241 
242 #if defined(__xpv)
243 	/*
244 	 * XXPV	Should probably think some more about how we deal
245 	 *	with panicing before it's really safe to panic.
246 	 *	On hypervisors, we reboot very quickly..  Perhaps panic
247 	 *	should only attempt to recover by rebooting if,
248 	 *	say, we were able to mount the root filesystem,
249 	 *	or if we successfully launched init(1m).
250 	 */
251 	if (panicstr && proc_init == NULL)
252 		(void) HYPERVISOR_shutdown(SHUTDOWN_poweroff);
253 #endif
254 	/*
255 	 * stop other cpus and raise our priority.  since there is only
256 	 * one active cpu after this, and our priority will be too high
257 	 * for us to be preempted, we're essentially single threaded
258 	 * from here on out.
259 	 */
260 	(void) spl6();
261 	if (!panicstr) {
262 		mutex_enter(&cpu_lock);
263 		pause_cpus(NULL);
264 		mutex_exit(&cpu_lock);
265 	}
266 
267 	/*
268 	 * If the system is panicking, the preloaded kernel is valid, and
269 	 * fastreboot_onpanic has been set, and the system has been up for
270 	 * longer than fastreboot_onpanic_uptime (default to 10 minutes),
271 	 * choose Fast Reboot.
272 	 */
273 	if (fcn == AD_BOOT && panicstr && newkernel.fi_valid &&
274 	    fastreboot_onpanic &&
275 	    (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) {
276 		fcn = AD_FASTREBOOT;
277 	}
278 
279 	/*
280 	 * Try to quiesce devices.
281 	 */
282 	if (is_first_quiesce) {
283 		/*
284 		 * Clear is_first_quiesce before calling quiesce_devices()
285 		 * so that if quiesce_devices() causes panics, it will not
286 		 * be invoked again.
287 		 */
288 		is_first_quiesce = 0;
289 
290 		quiesce_active = 1;
291 		quiesce_devices(ddi_root_node(), &reset_status);
292 		if (reset_status == -1) {
293 			if (fcn == AD_FASTREBOOT && !force_fastreboot) {
294 				prom_printf("Driver(s) not capable of fast "
295 				    "reboot.\n");
296 				prom_printf(fallback_str);
297 				fastreboot_capable = 0;
298 				fcn = AD_BOOT;
299 			} else if (fcn != AD_FASTREBOOT)
300 				fastreboot_capable = 0;
301 		}
302 		quiesce_active = 0;
303 	}
304 
305 	/*
306 	 * Try to reset devices. reset_leaves() should only be called
307 	 * a) when there are no other threads that could be accessing devices,
308 	 *    and
309 	 * b) on a system that's not capable of fast reboot (fastreboot_capable
310 	 *    being 0), or on a system where quiesce_devices() failed to
311 	 *    complete (quiesce_active being 1).
312 	 */
313 	if (is_first_reset && (!fastreboot_capable || quiesce_active)) {
314 		/*
315 		 * Clear is_first_reset before calling reset_devices()
316 		 * so that if reset_devices() causes panics, it will not
317 		 * be invoked again.
318 		 */
319 		is_first_reset = 0;
320 		reset_leaves();
321 	}
322 
323 	/* Verify newkernel checksum */
324 	if (fastreboot_capable && fcn == AD_FASTREBOOT &&
325 	    fastboot_cksum_verify(&newkernel) != 0) {
326 		fastreboot_capable = 0;
327 		prom_printf("Fast reboot: checksum failed for the new "
328 		    "kernel.\n");
329 		prom_printf(fallback_str);
330 	}
331 
332 	(void) spl8();
333 
334 	if (fastreboot_capable && fcn == AD_FASTREBOOT) {
335 		/*
336 		 * psm_shutdown is called within fast_reboot()
337 		 */
338 		fast_reboot();
339 	} else {
340 		(*psm_shutdownf)(cmd, fcn);
341 
342 		if (fcn == AD_HALT || fcn == AD_POWEROFF)
343 			halt((char *)NULL);
344 		else
345 			prom_reboot("");
346 	}
347 	/*NOTREACHED*/
348 }
349 
350 /* mdpreboot - may be called prior to mdboot while root fs still mounted */
351 /*ARGSUSED*/
352 void
353 mdpreboot(int cmd, int fcn, char *mdep)
354 {
355 	if (fcn == AD_FASTREBOOT && !fastreboot_capable) {
356 		fcn = AD_BOOT;
357 #ifdef	__xpv
358 		cmn_err(CE_WARN, "Fast reboot is not supported on xVM");
359 #else
360 		cmn_err(CE_WARN,
361 		    "Fast reboot is not supported on this platform");
362 #endif
363 	}
364 
365 	if (fcn == AD_FASTREBOOT) {
366 		fastboot_load_kernel(mdep);
367 		if (!newkernel.fi_valid)
368 			fcn = AD_BOOT;
369 	}
370 
371 	(*psm_preshutdownf)(cmd, fcn);
372 }
373 
374 static void
375 stop_other_cpus(void)
376 {
377 	ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */
378 	cpuset_t xcset;
379 
380 	CPUSET_ALL_BUT(xcset, CPU->cpu_id);
381 	xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)mach_cpu_halt);
382 	restore_int_flag(s);
383 }
384 
385 /*
386  *	Machine dependent abort sequence handling
387  */
388 void
389 abort_sequence_enter(char *msg)
390 {
391 	if (abort_enable == 0) {
392 		if (audit_active)
393 			audit_enterprom(0);
394 		return;
395 	}
396 	if (audit_active)
397 		audit_enterprom(1);
398 	debug_enter(msg);
399 	if (audit_active)
400 		audit_exitprom(1);
401 }
402 
403 /*
404  * Enter debugger.  Called when the user types ctrl-alt-d or whenever
405  * code wants to enter the debugger and possibly resume later.
406  */
407 void
408 debug_enter(
409 	char	*msg)		/* message to print, possibly NULL */
410 {
411 	if (dtrace_debugger_init != NULL)
412 		(*dtrace_debugger_init)();
413 
414 	if (msg)
415 		prom_printf("%s\n", msg);
416 
417 	if (boothowto & RB_DEBUG)
418 		kmdb_enter();
419 
420 	if (dtrace_debugger_fini != NULL)
421 		(*dtrace_debugger_fini)();
422 }
423 
424 void
425 reset(void)
426 {
427 	extern	void acpi_reset_system();
428 #if !defined(__xpv)
429 	ushort_t *bios_memchk;
430 
431 	/*
432 	 * Can't use psm_map_phys or acpi_reset_system before the hat is
433 	 * initialized.
434 	 */
435 	if (khat_running) {
436 		bios_memchk = (ushort_t *)psm_map_phys(0x472,
437 		    sizeof (ushort_t), PROT_READ | PROT_WRITE);
438 		if (bios_memchk)
439 			*bios_memchk = 0x1234;	/* bios memory check disable */
440 
441 		if (options_dip != NULL &&
442 		    ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0,
443 		    "efi-systab")) {
444 			efi_reset();
445 		}
446 
447 		/*
448 		 * The problem with using stubs is that we can call
449 		 * acpi_reset_system only after the kernel is up and running.
450 		 *
451 		 * We should create a global state to keep track of how far
452 		 * up the kernel is but for the time being we will depend on
453 		 * bootops. bootops cleared in startup_end().
454 		 */
455 		if (bootops == NULL)
456 			acpi_reset_system();
457 	}
458 
459 	pc_reset();
460 #else
461 	if (IN_XPV_PANIC()) {
462 		if (khat_running && bootops == NULL) {
463 			acpi_reset_system();
464 		}
465 
466 		pc_reset();
467 	}
468 
469 	(void) HYPERVISOR_shutdown(SHUTDOWN_reboot);
470 	panic("HYPERVISOR_shutdown() failed");
471 #endif
472 	/*NOTREACHED*/
473 }
474 
475 /*
476  * Halt the machine and return to the monitor
477  */
478 void
479 halt(char *s)
480 {
481 	stop_other_cpus();	/* send stop signal to other CPUs */
482 	if (s)
483 		prom_printf("(%s) \n", s);
484 	prom_exit_to_mon();
485 	/*NOTREACHED*/
486 }
487 
488 /*
489  * Initiate interrupt redistribution.
490  */
491 void
492 i_ddi_intr_redist_all_cpus()
493 {
494 }
495 
496 /*
497  * XXX These probably ought to live somewhere else
498  * XXX They are called from mem.c
499  */
500 
501 /*
502  * Convert page frame number to an OBMEM page frame number
503  * (i.e. put in the type bits -- zero for this implementation)
504  */
505 pfn_t
506 impl_obmem_pfnum(pfn_t pf)
507 {
508 	return (pf);
509 }
510 
511 #ifdef	NM_DEBUG
512 int nmi_test = 0;	/* checked in intentry.s during clock int */
513 int nmtest = -1;
514 nmfunc1(arg, rp)
515 int	arg;
516 struct regs *rp;
517 {
518 	printf("nmi called with arg = %x, regs = %x\n", arg, rp);
519 	nmtest += 50;
520 	if (arg == nmtest) {
521 		printf("ip = %x\n", rp->r_pc);
522 		return (1);
523 	}
524 	return (0);
525 }
526 
527 #endif
528 
529 #include <sys/bootsvcs.h>
530 
531 /* Hacked up initialization for initial kernel check out is HERE. */
532 /* The basic steps are: */
533 /*	kernel bootfuncs definition/initialization for KADB */
534 /*	kadb bootfuncs pointer initialization */
535 /*	putchar/getchar (interrupts disabled) */
536 
537 /* kadb bootfuncs pointer initialization */
538 
539 int
540 sysp_getchar()
541 {
542 	int i;
543 	ulong_t s;
544 
545 	if (cons_polledio == NULL) {
546 		/* Uh oh */
547 		prom_printf("getchar called with no console\n");
548 		for (;;)
549 			/* LOOP FOREVER */;
550 	}
551 
552 	s = clear_int_flag();
553 	i = cons_polledio->cons_polledio_getchar(
554 	    cons_polledio->cons_polledio_argument);
555 	restore_int_flag(s);
556 	return (i);
557 }
558 
559 void
560 sysp_putchar(int c)
561 {
562 	ulong_t s;
563 
564 	/*
565 	 * We have no alternative but to drop the output on the floor.
566 	 */
567 	if (cons_polledio == NULL ||
568 	    cons_polledio->cons_polledio_putchar == NULL)
569 		return;
570 
571 	s = clear_int_flag();
572 	cons_polledio->cons_polledio_putchar(
573 	    cons_polledio->cons_polledio_argument, c);
574 	restore_int_flag(s);
575 }
576 
577 int
578 sysp_ischar()
579 {
580 	int i;
581 	ulong_t s;
582 
583 	if (cons_polledio == NULL ||
584 	    cons_polledio->cons_polledio_ischar == NULL)
585 		return (0);
586 
587 	s = clear_int_flag();
588 	i = cons_polledio->cons_polledio_ischar(
589 	    cons_polledio->cons_polledio_argument);
590 	restore_int_flag(s);
591 	return (i);
592 }
593 
594 int
595 goany(void)
596 {
597 	prom_printf("Type any key to continue ");
598 	(void) prom_getchar();
599 	prom_printf("\n");
600 	return (1);
601 }
602 
603 static struct boot_syscalls kern_sysp = {
604 	sysp_getchar,	/*	unchar	(*getchar)();	7  */
605 	sysp_putchar,	/*	int	(*putchar)();	8  */
606 	sysp_ischar,	/*	int	(*ischar)();	9  */
607 };
608 
609 #if defined(__xpv)
610 int using_kern_polledio;
611 #endif
612 
613 void
614 kadb_uses_kernel()
615 {
616 	/*
617 	 * This routine is now totally misnamed, since it does not in fact
618 	 * control kadb's I/O; it only controls the kernel's prom_* I/O.
619 	 */
620 	sysp = &kern_sysp;
621 #if defined(__xpv)
622 	using_kern_polledio = 1;
623 #endif
624 }
625 
626 /*
627  *	the interface to the outside world
628  */
629 
630 /*
631  * poll_port -- wait for a register to achieve a
632  *		specific state.  Arguments are a mask of bits we care about,
633  *		and two sub-masks.  To return normally, all the bits in the
634  *		first sub-mask must be ON, all the bits in the second sub-
635  *		mask must be OFF.  If about seconds pass without the register
636  *		achieving the desired bit configuration, we return 1, else
637  *		0.
638  */
639 int
640 poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits)
641 {
642 	int i;
643 	ushort_t maskval;
644 
645 	for (i = 500000; i; i--) {
646 		maskval = inb(port) & mask;
647 		if (((maskval & onbits) == onbits) &&
648 		    ((maskval & offbits) == 0))
649 			return (0);
650 		drv_usecwait(10);
651 	}
652 	return (1);
653 }
654 
655 /*
656  * set_idle_cpu is called from idle() when a CPU becomes idle.
657  */
658 /*LINTED: static unused */
659 static uint_t last_idle_cpu;
660 
661 /*ARGSUSED*/
662 void
663 set_idle_cpu(int cpun)
664 {
665 	last_idle_cpu = cpun;
666 	(*psm_set_idle_cpuf)(cpun);
667 }
668 
669 /*
670  * unset_idle_cpu is called from idle() when a CPU is no longer idle.
671  */
672 /*ARGSUSED*/
673 void
674 unset_idle_cpu(int cpun)
675 {
676 	(*psm_unset_idle_cpuf)(cpun);
677 }
678 
679 /*
680  * This routine is almost correct now, but not quite.  It still needs the
681  * equivalent concept of "hres_last_tick", just like on the sparc side.
682  * The idea is to take a snapshot of the hi-res timer while doing the
683  * hrestime_adj updates under hres_lock in locore, so that the small
684  * interval between interrupt assertion and interrupt processing is
685  * accounted for correctly.  Once we have this, the code below should
686  * be modified to subtract off hres_last_tick rather than hrtime_base.
687  *
688  * I'd have done this myself, but I don't have source to all of the
689  * vendor-specific hi-res timer routines (grrr...).  The generic hook I
690  * need is something like "gethrtime_unlocked()", which would be just like
691  * gethrtime() but would assume that you're already holding CLOCK_LOCK().
692  * This is what the GET_HRTIME() macro is for on sparc (although it also
693  * serves the function of making time available without a function call
694  * so you don't take a register window overflow while traps are disabled).
695  */
696 void
697 pc_gethrestime(timestruc_t *tp)
698 {
699 	int lock_prev;
700 	timestruc_t now;
701 	int nslt;		/* nsec since last tick */
702 	int adj;		/* amount of adjustment to apply */
703 
704 loop:
705 	lock_prev = hres_lock;
706 	now = hrestime;
707 	nslt = (int)(gethrtime() - hres_last_tick);
708 	if (nslt < 0) {
709 		/*
710 		 * nslt < 0 means a tick came between sampling
711 		 * gethrtime() and hres_last_tick; restart the loop
712 		 */
713 
714 		goto loop;
715 	}
716 	now.tv_nsec += nslt;
717 	if (hrestime_adj != 0) {
718 		if (hrestime_adj > 0) {
719 			adj = (nslt >> ADJ_SHIFT);
720 			if (adj > hrestime_adj)
721 				adj = (int)hrestime_adj;
722 		} else {
723 			adj = -(nslt >> ADJ_SHIFT);
724 			if (adj < hrestime_adj)
725 				adj = (int)hrestime_adj;
726 		}
727 		now.tv_nsec += adj;
728 	}
729 	while ((unsigned long)now.tv_nsec >= NANOSEC) {
730 
731 		/*
732 		 * We might have a large adjustment or have been in the
733 		 * debugger for a long time; take care of (at most) four
734 		 * of those missed seconds (tv_nsec is 32 bits, so
735 		 * anything >4s will be wrapping around).  However,
736 		 * anything more than 2 seconds out of sync will trigger
737 		 * timedelta from clock() to go correct the time anyway,
738 		 * so do what we can, and let the big crowbar do the
739 		 * rest.  A similar correction while loop exists inside
740 		 * hres_tick(); in all cases we'd like tv_nsec to
741 		 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing
742 		 * user processes, but if tv_sec's a little behind for a
743 		 * little while, that's OK; time still monotonically
744 		 * increases.
745 		 */
746 
747 		now.tv_nsec -= NANOSEC;
748 		now.tv_sec++;
749 	}
750 	if ((hres_lock & ~1) != lock_prev)
751 		goto loop;
752 
753 	*tp = now;
754 }
755 
756 void
757 gethrestime_lasttick(timespec_t *tp)
758 {
759 	int s;
760 
761 	s = hr_clock_lock();
762 	*tp = hrestime;
763 	hr_clock_unlock(s);
764 }
765 
766 time_t
767 gethrestime_sec(void)
768 {
769 	timestruc_t now;
770 
771 	gethrestime(&now);
772 	return (now.tv_sec);
773 }
774 
775 /*
776  * Initialize a kernel thread's stack
777  */
778 
779 caddr_t
780 thread_stk_init(caddr_t stk)
781 {
782 	ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0);
783 	return (stk - SA(MINFRAME));
784 }
785 
786 /*
787  * Initialize lwp's kernel stack.
788  */
789 
790 #ifdef TRAPTRACE
791 /*
792  * There's a tricky interdependency here between use of sysenter and
793  * TRAPTRACE which needs recording to avoid future confusion (this is
794  * about the third time I've re-figured this out ..)
795  *
796  * Here's how debugging lcall works with TRAPTRACE.
797  *
798  * 1 We're in userland with a breakpoint on the lcall instruction.
799  * 2 We execute the instruction - the instruction pushes the userland
800  *   %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel
801  *   via the call gate.
802  * 3 The hardware raises a debug trap in kernel mode, the hardware
803  *   pushes %efl, %cs, %eip and gets to dbgtrap via the idt.
804  * 4 dbgtrap pushes the error code and trapno and calls cmntrap
805  * 5 cmntrap finishes building a trap frame
806  * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk
807  *   off the stack into the traptrace buffer.
808  *
809  * This means that the traptrace buffer contains the wrong values in
810  * %esp and %ss, but everything else in there is correct.
811  *
812  * Here's how debugging sysenter works with TRAPTRACE.
813  *
814  * a We're in userland with a breakpoint on the sysenter instruction.
815  * b We execute the instruction - the instruction pushes -nothing-
816  *   on the stack, but sets %cs, %eip, %ss, %esp to prearranged
817  *   values to take us to sys_sysenter, at the top of the lwp's
818  *   stack.
819  * c goto 3
820  *
821  * At this point, because we got into the kernel without the requisite
822  * five pushes on the stack, if we didn't make extra room, we'd
823  * end up with the TRACE_REGS macro fetching the saved %ss and %esp
824  * values from negative (unmapped) stack addresses -- which really bites.
825  * That's why we do the '-= 8' below.
826  *
827  * XXX	Note that reading "up" lwp0's stack works because t0 is declared
828  *	right next to t0stack in locore.s
829  */
830 #endif
831 
832 caddr_t
833 lwp_stk_init(klwp_t *lwp, caddr_t stk)
834 {
835 	caddr_t oldstk;
836 	struct pcb *pcb = &lwp->lwp_pcb;
837 
838 	oldstk = stk;
839 	stk -= SA(sizeof (struct regs) + SA(MINFRAME));
840 #ifdef TRAPTRACE
841 	stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */
842 #endif
843 	stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul));
844 	bzero(stk, oldstk - stk);
845 	lwp->lwp_regs = (void *)(stk + SA(MINFRAME));
846 
847 	/*
848 	 * Arrange that the virtualized %fs and %gs GDT descriptors
849 	 * have a well-defined initial state (present, ring 3
850 	 * and of type data).
851 	 */
852 #if defined(__amd64)
853 	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE)
854 		pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
855 	else
856 		pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;
857 #elif defined(__i386)
858 	pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;
859 #endif	/* __i386 */
860 	lwp_installctx(lwp);
861 	return (stk);
862 }
863 
864 /*ARGSUSED*/
865 void
866 lwp_stk_fini(klwp_t *lwp)
867 {}
868 
869 /*
870  * If we're not the panic CPU, we wait in panic_idle for reboot.
871  */
872 void
873 panic_idle(void)
874 {
875 	splx(ipltospl(CLOCK_LEVEL));
876 	(void) setjmp(&curthread->t_pcb);
877 
878 	dumpsys_helper();
879 
880 #ifndef __xpv
881 	for (;;)
882 		i86_halt();
883 #else
884 	for (;;)
885 		;
886 #endif
887 }
888 
889 /*
890  * Stop the other CPUs by cross-calling them and forcing them to enter
891  * the panic_idle() loop above.
892  */
893 /*ARGSUSED*/
894 void
895 panic_stopcpus(cpu_t *cp, kthread_t *t, int spl)
896 {
897 	processorid_t i;
898 	cpuset_t xcset;
899 
900 	/*
901 	 * In the case of a Xen panic, the hypervisor has already stopped
902 	 * all of the CPUs.
903 	 */
904 	if (!IN_XPV_PANIC()) {
905 		(void) splzs();
906 
907 		CPUSET_ALL_BUT(xcset, cp->cpu_id);
908 		xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle);
909 	}
910 
911 	for (i = 0; i < NCPU; i++) {
912 		if (i != cp->cpu_id && cpu[i] != NULL &&
913 		    (cpu[i]->cpu_flags & CPU_EXISTS))
914 			cpu[i]->cpu_flags |= CPU_QUIESCED;
915 	}
916 }
917 
918 /*
919  * Platform callback following each entry to panicsys().
920  */
921 /*ARGSUSED*/
922 void
923 panic_enter_hw(int spl)
924 {
925 	/* Nothing to do here */
926 }
927 
928 /*
929  * Platform-specific code to execute after panicstr is set: we invoke
930  * the PSM entry point to indicate that a panic has occurred.
931  */
932 /*ARGSUSED*/
933 void
934 panic_quiesce_hw(panic_data_t *pdp)
935 {
936 	psm_notifyf(PSM_PANIC_ENTER);
937 
938 	cmi_panic_callback();
939 
940 #ifdef	TRAPTRACE
941 	/*
942 	 * Turn off TRAPTRACE
943 	 */
944 	TRAPTRACE_FREEZE;
945 #endif	/* TRAPTRACE */
946 }
947 
948 /*
949  * Platform callback prior to writing crash dump.
950  */
951 /*ARGSUSED*/
952 void
953 panic_dump_hw(int spl)
954 {
955 	/* Nothing to do here */
956 }
957 
958 void *
959 plat_traceback(void *fpreg)
960 {
961 #ifdef __xpv
962 	if (IN_XPV_PANIC())
963 		return (xpv_traceback(fpreg));
964 #endif
965 	return (fpreg);
966 }
967 
968 /*ARGSUSED*/
969 void
970 plat_tod_fault(enum tod_fault_type tod_bad)
971 {}
972 
973 /*ARGSUSED*/
974 int
975 blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class)
976 {
977 	return (ENOTSUP);
978 }
979 
980 /*
981  * The underlying console output routines are protected by raising IPL in case
982  * we are still calling into the early boot services.  Once we start calling
983  * the kernel console emulator, it will disable interrupts completely during
984  * character rendering (see sysp_putchar, for example).  Refer to the comments
985  * and code in common/os/console.c for more information on these callbacks.
986  */
987 /*ARGSUSED*/
988 int
989 console_enter(int busy)
990 {
991 	return (splzs());
992 }
993 
994 /*ARGSUSED*/
995 void
996 console_exit(int busy, int spl)
997 {
998 	splx(spl);
999 }
1000 
1001 /*
1002  * Allocate a region of virtual address space, unmapped.
1003  * Stubbed out except on sparc, at least for now.
1004  */
1005 /*ARGSUSED*/
1006 void *
1007 boot_virt_alloc(void *addr, size_t size)
1008 {
1009 	return (addr);
1010 }
1011 
1012 volatile unsigned long	tenmicrodata;
1013 
1014 void
1015 tenmicrosec(void)
1016 {
1017 	extern int gethrtime_hires;
1018 
1019 	if (gethrtime_hires) {
1020 		hrtime_t start, end;
1021 		start = end =  gethrtime();
1022 		while ((end - start) < (10 * (NANOSEC / MICROSEC))) {
1023 			SMT_PAUSE();
1024 			end = gethrtime();
1025 		}
1026 	} else {
1027 #if defined(__xpv)
1028 		hrtime_t newtime;
1029 
1030 		newtime = xpv_gethrtime() + 10000; /* now + 10 us */
1031 		while (xpv_gethrtime() < newtime)
1032 			SMT_PAUSE();
1033 #else	/* __xpv */
1034 		int i;
1035 
1036 		/*
1037 		 * Artificial loop to induce delay.
1038 		 */
1039 		for (i = 0; i < microdata; i++)
1040 			tenmicrodata = microdata;
1041 #endif	/* __xpv */
1042 	}
1043 }
1044 
1045 /*
1046  * get_cpu_mstate() is passed an array of timestamps, NCMSTATES
1047  * long, and it fills in the array with the time spent on cpu in
1048  * each of the mstates, where time is returned in nsec.
1049  *
1050  * No guarantee is made that the returned values in times[] will
1051  * monotonically increase on sequential calls, although this will
1052  * be true in the long run. Any such guarantee must be handled by
1053  * the caller, if needed. This can happen if we fail to account
1054  * for elapsed time due to a generation counter conflict, yet we
1055  * did account for it on a prior call (see below).
1056  *
1057  * The complication is that the cpu in question may be updating
1058  * its microstate at the same time that we are reading it.
1059  * Because the microstate is only updated when the CPU's state
1060  * changes, the values in cpu_intracct[] can be indefinitely out
1061  * of date. To determine true current values, it is necessary to
1062  * compare the current time with cpu_mstate_start, and add the
1063  * difference to times[cpu_mstate].
1064  *
1065  * This can be a problem if those values are changing out from
1066  * under us. Because the code path in new_cpu_mstate() is
1067  * performance critical, we have not added a lock to it. Instead,
1068  * we have added a generation counter. Before beginning
1069  * modifications, the counter is set to 0. After modifications,
1070  * it is set to the old value plus one.
1071  *
1072  * get_cpu_mstate() will not consider the values of cpu_mstate
1073  * and cpu_mstate_start to be usable unless the value of
1074  * cpu_mstate_gen is both non-zero and unchanged, both before and
1075  * after reading the mstate information. Note that we must
1076  * protect against out-of-order loads around accesses to the
1077  * generation counter. Also, this is a best effort approach in
1078  * that we do not retry should the counter be found to have
1079  * changed.
1080  *
1081  * cpu_intracct[] is used to identify time spent in each CPU
1082  * mstate while handling interrupts. Such time should be reported
1083  * against system time, and so is subtracted out from its
1084  * corresponding cpu_acct[] time and added to
1085  * cpu_acct[CMS_SYSTEM].
1086  */
1087 
1088 void
1089 get_cpu_mstate(cpu_t *cpu, hrtime_t *times)
1090 {
1091 	int i;
1092 	hrtime_t now, start;
1093 	uint16_t gen;
1094 	uint16_t state;
1095 	hrtime_t intracct[NCMSTATES];
1096 
1097 	/*
1098 	 * Load all volatile state under the protection of membar.
1099 	 * cpu_acct[cpu_mstate] must be loaded to avoid double counting
1100 	 * of (now - cpu_mstate_start) by a change in CPU mstate that
1101 	 * arrives after we make our last check of cpu_mstate_gen.
1102 	 */
1103 
1104 	now = gethrtime_unscaled();
1105 	gen = cpu->cpu_mstate_gen;
1106 
1107 	membar_consumer();	/* guarantee load ordering */
1108 	start = cpu->cpu_mstate_start;
1109 	state = cpu->cpu_mstate;
1110 	for (i = 0; i < NCMSTATES; i++) {
1111 		intracct[i] = cpu->cpu_intracct[i];
1112 		times[i] = cpu->cpu_acct[i];
1113 	}
1114 	membar_consumer();	/* guarantee load ordering */
1115 
1116 	if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start)
1117 		times[state] += now - start;
1118 
1119 	for (i = 0; i < NCMSTATES; i++) {
1120 		if (i == CMS_SYSTEM)
1121 			continue;
1122 		times[i] -= intracct[i];
1123 		if (times[i] < 0) {
1124 			intracct[i] += times[i];
1125 			times[i] = 0;
1126 		}
1127 		times[CMS_SYSTEM] += intracct[i];
1128 		scalehrtime(&times[i]);
1129 	}
1130 	scalehrtime(&times[CMS_SYSTEM]);
1131 }
1132 
1133 /*
1134  * This is a version of the rdmsr instruction that allows
1135  * an error code to be returned in the case of failure.
1136  */
1137 int
1138 checked_rdmsr(uint_t msr, uint64_t *value)
1139 {
1140 	if ((x86_feature & X86_MSR) == 0)
1141 		return (ENOTSUP);
1142 	*value = rdmsr(msr);
1143 	return (0);
1144 }
1145 
1146 /*
1147  * This is a version of the wrmsr instruction that allows
1148  * an error code to be returned in the case of failure.
1149  */
1150 int
1151 checked_wrmsr(uint_t msr, uint64_t value)
1152 {
1153 	if ((x86_feature & X86_MSR) == 0)
1154 		return (ENOTSUP);
1155 	wrmsr(msr, value);
1156 	return (0);
1157 }
1158 
1159 /*
1160  * The mem driver's usual method of using hat_devload() to establish a
1161  * temporary mapping will not work for foreign pages mapped into this
1162  * domain or for the special hypervisor-provided pages.  For the foreign
1163  * pages, we often don't know which domain owns them, so we can't ask the
1164  * hypervisor to set up a new mapping.  For the other pages, we don't have
1165  * a pfn, so we can't create a new PTE.  For these special cases, we do a
1166  * direct uiomove() from the existing kernel virtual address.
1167  */
1168 /*ARGSUSED*/
1169 int
1170 plat_mem_do_mmio(struct uio *uio, enum uio_rw rw)
1171 {
1172 #if defined(__xpv)
1173 	void *va = (void *)(uintptr_t)uio->uio_loffset;
1174 	off_t pageoff = uio->uio_loffset & PAGEOFFSET;
1175 	size_t nbytes = MIN((size_t)(PAGESIZE - pageoff),
1176 	    (size_t)uio->uio_iov->iov_len);
1177 
1178 	if ((rw == UIO_READ &&
1179 	    (va == HYPERVISOR_shared_info || va == xen_info)) ||
1180 	    (pfn_is_foreign(hat_getpfnum(kas.a_hat, va))))
1181 		return (uiomove(va, nbytes, rw, uio));
1182 #endif
1183 	return (ENOTSUP);
1184 }
1185 
1186 pgcnt_t
1187 num_phys_pages()
1188 {
1189 	pgcnt_t npages = 0;
1190 	struct memlist *mp;
1191 
1192 #if defined(__xpv)
1193 	if (DOMAIN_IS_INITDOMAIN(xen_info))
1194 		return (xpv_nr_phys_pages());
1195 #endif /* __xpv */
1196 
1197 	for (mp = phys_install; mp != NULL; mp = mp->next)
1198 		npages += mp->size >> PAGESHIFT;
1199 
1200 	return (npages);
1201 }
1202 
1203 /* cpu threshold for compressed dumps */
1204 #ifdef _LP64
1205 uint_t dump_plat_mincpu = DUMP_PLAT_X86_64_MINCPU;
1206 #else
1207 uint_t dump_plat_mincpu = DUMP_PLAT_X86_32_MINCPU;
1208 #endif
1209 
1210 int
1211 dump_plat_addr()
1212 {
1213 #ifdef __xpv
1214 	pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1215 	mem_vtop_t mem_vtop;
1216 	int cnt;
1217 
1218 	/*
1219 	 * On the hypervisor, we want to dump the page with shared_info on it.
1220 	 */
1221 	if (!IN_XPV_PANIC()) {
1222 		mem_vtop.m_as = &kas;
1223 		mem_vtop.m_va = HYPERVISOR_shared_info;
1224 		mem_vtop.m_pfn = pfn;
1225 		dumpvp_write(&mem_vtop, sizeof (mem_vtop_t));
1226 		cnt = 1;
1227 	} else {
1228 		cnt = dump_xpv_addr();
1229 	}
1230 	return (cnt);
1231 #else
1232 	return (0);
1233 #endif
1234 }
1235 
1236 void
1237 dump_plat_pfn()
1238 {
1239 #ifdef __xpv
1240 	pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN;
1241 
1242 	if (!IN_XPV_PANIC())
1243 		dumpvp_write(&pfn, sizeof (pfn));
1244 	else
1245 		dump_xpv_pfn();
1246 #endif
1247 }
1248 
1249 /*ARGSUSED*/
1250 int
1251 dump_plat_data(void *dump_cbuf)
1252 {
1253 #ifdef __xpv
1254 	uint32_t csize;
1255 	int cnt;
1256 
1257 	if (!IN_XPV_PANIC()) {
1258 		csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf,
1259 		    PAGESIZE);
1260 		dumpvp_write(&csize, sizeof (uint32_t));
1261 		dumpvp_write(dump_cbuf, csize);
1262 		cnt = 1;
1263 	} else {
1264 		cnt = dump_xpv_data(dump_cbuf);
1265 	}
1266 	return (cnt);
1267 #else
1268 	return (0);
1269 #endif
1270 }
1271 
1272 /*
1273  * Calculates a linear address, given the CS selector and PC values
1274  * by looking up the %cs selector process's LDT or the CPU's GDT.
1275  * proc->p_ldtlock must be held across this call.
1276  */
1277 int
1278 linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1279 {
1280 	user_desc_t	*descrp;
1281 	caddr_t		baseaddr;
1282 	uint16_t	idx = SELTOIDX(rp->r_cs);
1283 
1284 	ASSERT(rp->r_cs <= 0xFFFF);
1285 	ASSERT(MUTEX_HELD(&p->p_ldtlock));
1286 
1287 	if (SELISLDT(rp->r_cs)) {
1288 		/*
1289 		 * Currently 64 bit processes cannot have private LDTs.
1290 		 */
1291 		ASSERT(p->p_model != DATAMODEL_LP64);
1292 
1293 		if (p->p_ldt == NULL)
1294 			return (-1);
1295 
1296 		descrp = &p->p_ldt[idx];
1297 		baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1298 
1299 		/*
1300 		 * Calculate the linear address (wraparound is not only ok,
1301 		 * it's expected behavior).  The cast to uint32_t is because
1302 		 * LDT selectors are only allowed in 32-bit processes.
1303 		 */
1304 		*linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1305 		    rp->r_pc);
1306 	} else {
1307 #ifdef DEBUG
1308 		descrp = &CPU->cpu_gdt[idx];
1309 		baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1310 		/* GDT-based descriptors' base addresses should always be 0 */
1311 		ASSERT(baseaddr == 0);
1312 #endif
1313 		*linearp = (caddr_t)(uintptr_t)rp->r_pc;
1314 	}
1315 
1316 	return (0);
1317 }
1318 
1319 /*
1320  * The implementation of dtrace_linear_pc is similar to the that of
1321  * linear_pc, above, but here we acquire p_ldtlock before accessing
1322  * p_ldt.  This implementation is used by the pid provider; we prefix
1323  * it with "dtrace_" to avoid inducing spurious tracing events.
1324  */
1325 int
1326 dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp)
1327 {
1328 	user_desc_t	*descrp;
1329 	caddr_t		baseaddr;
1330 	uint16_t	idx = SELTOIDX(rp->r_cs);
1331 
1332 	ASSERT(rp->r_cs <= 0xFFFF);
1333 
1334 	if (SELISLDT(rp->r_cs)) {
1335 		/*
1336 		 * Currently 64 bit processes cannot have private LDTs.
1337 		 */
1338 		ASSERT(p->p_model != DATAMODEL_LP64);
1339 
1340 		mutex_enter(&p->p_ldtlock);
1341 		if (p->p_ldt == NULL) {
1342 			mutex_exit(&p->p_ldtlock);
1343 			return (-1);
1344 		}
1345 		descrp = &p->p_ldt[idx];
1346 		baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1347 		mutex_exit(&p->p_ldtlock);
1348 
1349 		/*
1350 		 * Calculate the linear address (wraparound is not only ok,
1351 		 * it's expected behavior).  The cast to uint32_t is because
1352 		 * LDT selectors are only allowed in 32-bit processes.
1353 		 */
1354 		*linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr +
1355 		    rp->r_pc);
1356 	} else {
1357 #ifdef DEBUG
1358 		descrp = &CPU->cpu_gdt[idx];
1359 		baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp);
1360 		/* GDT-based descriptors' base addresses should always be 0 */
1361 		ASSERT(baseaddr == 0);
1362 #endif
1363 		*linearp = (caddr_t)(uintptr_t)rp->r_pc;
1364 	}
1365 
1366 	return (0);
1367 }
1368