xref: /titanic_50/usr/src/uts/i86pc/os/cpr_impl.c (revision 22337b4b3c8a2b9db615e524e0e9e1fbd3dc71bf)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * Platform specific implementation code
30  * Currently only suspend to RAM is supported (ACPI S3)
31  */
32 
33 #define	SUNDDI_IMPL
34 
35 #include <sys/types.h>
36 #include <sys/promif.h>
37 #include <sys/prom_isa.h>
38 #include <sys/prom_plat.h>
39 #include <sys/cpuvar.h>
40 #include <sys/pte.h>
41 #include <vm/hat.h>
42 #include <vm/page.h>
43 #include <vm/as.h>
44 #include <sys/cpr.h>
45 #include <sys/kmem.h>
46 #include <sys/clock.h>
47 #include <sys/kmem.h>
48 #include <sys/panic.h>
49 #include <vm/seg_kmem.h>
50 #include <sys/cpu_module.h>
51 #include <sys/callb.h>
52 #include <sys/machsystm.h>
53 #include <sys/vmsystm.h>
54 #include <sys/systm.h>
55 #include <sys/archsystm.h>
56 #include <sys/stack.h>
57 #include <sys/fs/ufs_fs.h>
58 #include <sys/memlist.h>
59 #include <sys/bootconf.h>
60 #include <sys/thread.h>
61 #include <sys/x_call.h>
62 #include <sys/smp_impldefs.h>
63 #include <vm/vm_dep.h>
64 #include <sys/psm.h>
65 #include <sys/epm.h>
66 #include <sys/cpr_wakecode.h>
67 #include <sys/x86_archext.h>
68 #include <sys/reboot.h>
69 #include <sys/acpi/acpi.h>
70 #include <sys/acpica.h>
71 
72 #define	AFMT	"%lx"
73 
74 extern int	flushes_require_xcalls;
75 extern cpuset_t	cpu_ready_set;
76 
77 #if defined(__amd64)
78 extern void	*wc_long_mode_64(void);
79 #endif	/* __amd64 */
80 extern int	tsc_gethrtime_enable;
81 extern	void	i_cpr_start_cpu(void);
82 
83 ushort_t	cpr_mach_type = CPR_MACHTYPE_X86;
84 void		(*cpr_start_cpu_func)(void) = i_cpr_start_cpu;
85 
86 static wc_cpu_t	*wc_other_cpus = NULL;
87 static cpuset_t procset = 1;
88 
89 static void
90 init_real_mode_platter(int cpun, uint32_t offset, uint_t cr4, wc_desctbr_t gdt);
91 
92 static int i_cpr_platform_alloc(psm_state_request_t *req);
93 static void i_cpr_platform_free(psm_state_request_t *req);
94 static int i_cpr_save_apic(psm_state_request_t *req);
95 static int i_cpr_restore_apic(psm_state_request_t *req);
96 static int wait_for_set(cpuset_t *set, int who);
97 
98 #if defined(__amd64)
99 static void restore_stack(wc_cpu_t *cpup);
100 static void save_stack(wc_cpu_t *cpup);
101 void (*save_stack_func)(wc_cpu_t *) = save_stack;
102 #endif	/* __amd64 */
103 
104 /*
105  * restart paused slave cpus
106  */
107 void
108 i_cpr_machdep_setup(void)
109 {
110 	if (ncpus > 1) {
111 		CPR_DEBUG(CPR_DEBUG1, ("MP restarted...\n"));
112 		mutex_enter(&cpu_lock);
113 		start_cpus();
114 		mutex_exit(&cpu_lock);
115 	}
116 }
117 
118 
119 /*
120  * Stop all interrupt activities in the system
121  */
122 void
123 i_cpr_stop_intr(void)
124 {
125 	(void) spl7();
126 }
127 
128 /*
129  * Set machine up to take interrupts
130  */
131 void
132 i_cpr_enable_intr(void)
133 {
134 	(void) spl0();
135 }
136 
137 /*
138  * Save miscellaneous information which needs to be written to the
139  * state file.  This information is required to re-initialize
140  * kernel/prom handshaking.
141  */
142 void
143 i_cpr_save_machdep_info(void)
144 {
145 	int notcalled = 0;
146 	ASSERT(notcalled);
147 }
148 
149 
150 void
151 i_cpr_set_tbr(void)
152 {
153 }
154 
155 
156 processorid_t
157 i_cpr_bootcpuid(void)
158 {
159 	return (0);
160 }
161 
162 /*
163  * cpu0 should contain bootcpu info
164  */
165 cpu_t *
166 i_cpr_bootcpu(void)
167 {
168 	ASSERT(MUTEX_HELD(&cpu_lock));
169 
170 	return (cpu_get(i_cpr_bootcpuid()));
171 }
172 
173 /*
174  *	Save context for the specified CPU
175  */
176 void *
177 i_cpr_save_context(void *arg)
178 {
179 	long	index = (long)arg;
180 	psm_state_request_t *papic_state;
181 	int resuming;
182 	int	ret;
183 
184 	PMD(PMD_SX, ("i_cpr_save_context() index = %ld\n", index))
185 
186 	ASSERT(index < NCPU);
187 
188 	papic_state = &(wc_other_cpus + index)->wc_apic_state;
189 
190 	ret = i_cpr_platform_alloc(papic_state);
191 	ASSERT(ret == 0);
192 
193 	ret = i_cpr_save_apic(papic_state);
194 	ASSERT(ret == 0);
195 
196 	/*
197 	 * wc_save_context returns twice, once when susending and
198 	 * once when resuming,  wc_save_context() returns 0 when
199 	 * suspending and non-zero upon resume
200 	 */
201 	resuming = (wc_save_context(wc_other_cpus + index) == 0);
202 
203 	PMD(PMD_SX, ("i_cpr_save_context: wc_save_context returns %d\n",
204 	    resuming))
205 
206 	/*
207 	 * do NOT call any functions after this point, because doing so
208 	 * will modify the stack that we are running on
209 	 */
210 
211 	if (resuming) {
212 
213 		ret = i_cpr_restore_apic(papic_state);
214 		ASSERT(ret == 0);
215 
216 		i_cpr_platform_free(papic_state);
217 
218 		/*
219 		 * Setting the bit in cpu_ready_set must be the last operation
220 		 * in processor initialization; the boot CPU will continue to
221 		 * boot once it sees this bit set for all active CPUs.
222 		 */
223 		CPUSET_ATOMIC_ADD(cpu_ready_set, CPU->cpu_id);
224 
225 		PMD(PMD_SX,
226 		    ("cpu_release() cpu_ready_set = %lx, CPU->cpu_id = %d\n",
227 		    cpu_ready_set, CPU->cpu_id))
228 	}
229 	return (NULL);
230 }
231 
232 static ushort_t *warm_reset_vector = NULL;
233 
234 static ushort_t *
235 map_warm_reset_vector()
236 {
237 	/*LINTED*/
238 	if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR,
239 	    sizeof (ushort_t *), PROT_READ|PROT_WRITE)))
240 		return (NULL);
241 
242 	/*
243 	 * setup secondary cpu bios boot up vector
244 	 */
245 	*warm_reset_vector = (ushort_t)((caddr_t)
246 	    /*LINTED*/
247 	    ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va
248 	    + ((ulong_t)rm_platter_va & 0xf));
249 	warm_reset_vector++;
250 	*warm_reset_vector = (ushort_t)(rm_platter_pa >> 4);
251 
252 	--warm_reset_vector;
253 	return (warm_reset_vector);
254 }
255 
256 void
257 i_cpr_pre_resume_cpus()
258 {
259 	/*
260 	 * this is a cut down version of start_other_cpus()
261 	 * just do the initialization to wake the other cpus
262 	 */
263 	unsigned who;
264 	int boot_cpuid = i_cpr_bootcpuid();
265 	uint32_t		code_length = 0;
266 	caddr_t			wakevirt = rm_platter_va;
267 	/*LINTED*/
268 	wakecode_t		*wp = (wakecode_t *)wakevirt;
269 	char *str = "i_cpr_pre_resume_cpus";
270 	extern int get_tsc_ready();
271 	int err;
272 
273 	/*LINTED*/
274 	rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va;
275 
276 	/*
277 	 * Copy the real mode code at "real_mode_start" to the
278 	 * page at rm_platter_va.
279 	 */
280 	warm_reset_vector = map_warm_reset_vector();
281 	if (warm_reset_vector == NULL) {
282 		PMD(PMD_SX, ("i_cpr_pre_resume_cpus() returning #2\n"))
283 		return;
284 	}
285 
286 	flushes_require_xcalls = 1;
287 
288 	/*
289 	 * We lock our affinity to the master CPU to ensure that all slave CPUs
290 	 * do their TSC syncs with the same CPU.
291 	 */
292 
293 	affinity_set(CPU_CURRENT);
294 
295 	/*
296 	 * mark the boot cpu as being ready, since we are running on that cpu
297 	 */
298 	CPUSET_ONLY(cpu_ready_set, boot_cpuid);
299 
300 	for (who = 0; who < ncpus; who++) {
301 
302 		wc_cpu_t	*cpup = wc_other_cpus + who;
303 		wc_desctbr_t	gdt;
304 
305 		if (who == boot_cpuid)
306 			continue;
307 
308 		if (!CPU_IN_SET(mp_cpus, who))
309 			continue;
310 
311 		PMD(PMD_SX, ("%s() waking up %d cpu\n", str, who))
312 
313 		bcopy(cpup, &(wp->wc_cpu), sizeof (wc_cpu_t));
314 
315 		gdt.base = cpup->wc_gdt_base;
316 		gdt.limit = cpup->wc_gdt_limit;
317 
318 #if defined(__amd64)
319 		code_length = (uint32_t)wc_long_mode_64 - (uint32_t)wc_rm_start;
320 #else
321 		code_length = 0;
322 #endif
323 
324 		init_real_mode_platter(who, code_length, cpup->wc_cr4, gdt);
325 
326 		if ((err = mach_cpuid_start(who, rm_platter_va)) != 0) {
327 			cmn_err(CE_WARN, "cpu%d: failed to start during "
328 			    "suspend/resume error %d", who, err);
329 			continue;
330 		}
331 
332 		PMD(PMD_SX, ("%s() #1 waiting for procset 0x%lx\n", str,
333 		    (ulong_t)procset))
334 
335 		if (!wait_for_set(&procset, who))
336 			continue;
337 
338 		PMD(PMD_SX, ("%s() %d cpu started\n", str, who))
339 
340 		PMD(PMD_SX, ("%s() tsc_ready = %d\n", str, get_tsc_ready()))
341 
342 		if (tsc_gethrtime_enable) {
343 			PMD(PMD_SX, ("%s() calling tsc_sync_master\n", str))
344 			tsc_sync_master(who);
345 		}
346 
347 		PMD(PMD_SX, ("%s() waiting for cpu_ready_set %ld\n", str,
348 		    cpu_ready_set))
349 		/*
350 		 * Wait for cpu to declare that it is ready, we want the
351 		 * cpus to start serially instead of in parallel, so that
352 		 * they do not contend with each other in wc_rm_start()
353 		 */
354 		if (!wait_for_set(&cpu_ready_set, who))
355 			continue;
356 
357 		/*
358 		 * do not need to re-initialize dtrace using dtrace_cpu_init
359 		 * function
360 		 */
361 		PMD(PMD_SX, ("%s() cpu %d now ready\n", str, who))
362 	}
363 
364 	affinity_clear();
365 
366 	PMD(PMD_SX, ("%s() all cpus now ready\n", str))
367 
368 }
369 
370 static void
371 unmap_warm_reset_vector(ushort_t *warm_reset_vector)
372 {
373 	psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *));
374 }
375 
376 /*
377  * We need to setup a 1:1 (virtual to physical) mapping for the
378  * page containing the wakeup code.
379  */
380 static struct as *save_as;	/* when switching to kas */
381 
382 static void
383 unmap_wakeaddr_1to1(uint64_t wakephys)
384 {
385 	uintptr_t	wp = (uintptr_t)wakephys;
386 	hat_setup(save_as->a_hat, 0);	/* switch back from kernel hat */
387 	hat_unload(kas.a_hat, (caddr_t)wp, PAGESIZE, HAT_UNLOAD);
388 }
389 
390 void
391 i_cpr_post_resume_cpus()
392 {
393 	uint64_t	wakephys = rm_platter_pa;
394 
395 	if (warm_reset_vector != NULL)
396 		unmap_warm_reset_vector(warm_reset_vector);
397 
398 	hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE,
399 	    HAT_UNLOAD);
400 
401 	/*
402 	 * cmi_post_mpstartup() is only required upon boot not upon
403 	 * resume from RAM
404 	 */
405 
406 	PT(PT_UNDO1to1);
407 	/* Tear down 1:1 mapping for wakeup code */
408 	unmap_wakeaddr_1to1(wakephys);
409 }
410 
411 /* ARGSUSED */
412 void
413 i_cpr_handle_xc(int flag)
414 {
415 }
416 
417 int
418 i_cpr_reusable_supported(void)
419 {
420 	return (0);
421 }
422 static void
423 map_wakeaddr_1to1(uint64_t wakephys)
424 {
425 	uintptr_t	wp = (uintptr_t)wakephys;
426 	hat_devload(kas.a_hat, (caddr_t)wp, PAGESIZE, btop(wakephys),
427 	    (PROT_READ|PROT_WRITE|PROT_EXEC|HAT_STORECACHING_OK|HAT_NOSYNC),
428 	    HAT_LOAD);
429 	save_as = curthread->t_procp->p_as;
430 	hat_setup(kas.a_hat, 0);	/* switch to kernel-only hat */
431 }
432 
433 
434 void
435 prt_other_cpus()
436 {
437 	int	who;
438 
439 	if (ncpus == 1) {
440 		PMD(PMD_SX, ("prt_other_cpus() other cpu table empty for "
441 		    "uniprocessor machine\n"))
442 		return;
443 	}
444 
445 	for (who = 0; who < ncpus; who++) {
446 
447 		wc_cpu_t	*cpup = wc_other_cpus + who;
448 
449 		PMD(PMD_SX, ("prt_other_cpus() who = %d, gdt=%p:%x, "
450 		    "idt=%p:%x, ldt=%lx, tr=%lx, kgsbase="
451 		    AFMT ", sp=%lx\n", who,
452 		    (void *)cpup->wc_gdt_base, cpup->wc_gdt_limit,
453 		    (void *)cpup->wc_idt_base, cpup->wc_idt_limit,
454 		    (long)cpup->wc_ldt, (long)cpup->wc_tr,
455 		    (long)cpup->wc_kgsbase, (long)cpup->wc_rsp))
456 	}
457 }
458 
459 /*
460  * Power down the system.
461  */
462 int
463 i_cpr_power_down(int sleeptype)
464 {
465 	caddr_t		wakevirt = rm_platter_va;
466 	uint64_t	wakephys = rm_platter_pa;
467 	uint_t		saved_intr;
468 	uint32_t	code_length = 0;
469 	wc_desctbr_t	gdt;
470 	/*LINTED*/
471 	wakecode_t	*wp = (wakecode_t *)wakevirt;
472 	/*LINTED*/
473 	rm_platter_t	*wcpp = (rm_platter_t *)wakevirt;
474 	wc_cpu_t	*cpup = &(wp->wc_cpu);
475 	dev_info_t	*ppm;
476 	int		ret = 0;
477 	power_req_t	power_req;
478 	char *str =	"i_cpr_power_down";
479 #if defined(__amd64)
480 	/*LINTED*/
481 	rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va;
482 #endif
483 	extern int	cpr_suspend_succeeded;
484 	extern void	kernel_wc_code();
485 	extern ulong_t	intr_clear(void);
486 	extern void	intr_restore(ulong_t);
487 
488 	ASSERT(sleeptype == CPR_TORAM);
489 	ASSERT(CPU->cpu_id == 0);
490 
491 	if ((ppm = PPM(ddi_root_node())) == NULL) {
492 		PMD(PMD_SX, ("%s: root node not claimed\n", str))
493 		return (ENOTTY);
494 	}
495 
496 	PMD(PMD_SX, ("Entering %s()\n", str))
497 
498 	PT(PT_IC);
499 	saved_intr = intr_clear();
500 
501 	PT(PT_1to1);
502 	/* Setup 1:1 mapping for wakeup code */
503 	map_wakeaddr_1to1(wakephys);
504 
505 	PMD(PMD_SX, ("ncpus=%d\n", ncpus))
506 
507 	PMD(PMD_SX, ("wc_rm_end - wc_rm_start=%lx WC_CODESIZE=%x\n",
508 	    ((size_t)((uint_t)wc_rm_end - (uint_t)wc_rm_start)), WC_CODESIZE))
509 
510 	PMD(PMD_SX, ("wakevirt=%p, wakephys=%x\n",
511 	    (void *)wakevirt, (uint_t)wakephys))
512 
513 	ASSERT(((size_t)((uint_t)wc_rm_end - (uint_t)wc_rm_start)) <
514 	    WC_CODESIZE);
515 
516 	bzero(wakevirt, PAGESIZE);
517 
518 	/* Copy code to rm_platter */
519 	bcopy((caddr_t)wc_rm_start, wakevirt,
520 	    (size_t)((uint_t)wc_rm_end - (uint_t)wc_rm_start));
521 
522 	prt_other_cpus();
523 
524 #if defined(__amd64)
525 
526 	PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n",
527 	    (ulong_t)real_mode_platter->rm_cr4, (ulong_t)getcr4()))
528 	PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n",
529 	    (ulong_t)real_mode_platter->rm_pdbr, getcr3()))
530 
531 	real_mode_platter->rm_cr4 = getcr4();
532 	real_mode_platter->rm_pdbr = getcr3();
533 
534 	rmp_gdt_init(real_mode_platter);
535 
536 	/*
537 	 * Since the CPU needs to jump to protected mode using an identity
538 	 * mapped address, we need to calculate it here.
539 	 */
540 	real_mode_platter->rm_longmode64_addr = rm_platter_pa +
541 	    ((uint32_t)wc_long_mode_64 - (uint32_t)wc_rm_start);
542 
543 	PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n",
544 	    (ulong_t)real_mode_platter->rm_cr4, getcr4()))
545 
546 	PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n",
547 	    (ulong_t)real_mode_platter->rm_pdbr, getcr3()))
548 
549 	PMD(PMD_SX, ("real_mode_platter->rm_longmode64_addr=%lx\n",
550 	    (ulong_t)real_mode_platter->rm_longmode64_addr))
551 
552 #endif
553 
554 	PMD(PMD_SX, ("mp_cpus=%lx\n", (ulong_t)mp_cpus))
555 
556 	PT(PT_SC);
557 	if (wc_save_context(cpup)) {
558 
559 		ret = i_cpr_platform_alloc(&(wc_other_cpus->wc_apic_state));
560 		if (ret != 0)
561 			return (ret);
562 
563 		ret = i_cpr_save_apic(&(wc_other_cpus->wc_apic_state));
564 		PMD(PMD_SX, ("%s: i_cpr_save_apic() returned %d\n", str, ret))
565 		if (ret != 0)
566 			return (ret);
567 
568 		PMD(PMD_SX, ("wakephys=%x, kernel_wc_code=%p\n",
569 		    (uint_t)wakephys, (void *)&kernel_wc_code))
570 		PMD(PMD_SX, ("virtaddr=%lx, retaddr=%lx\n",
571 		    (long)cpup->wc_virtaddr, (long)cpup->wc_retaddr))
572 		PMD(PMD_SX, ("ebx=%x, edi=%x, esi=%x, ebp=%x, esp=%x\n",
573 		    cpup->wc_ebx, cpup->wc_edi, cpup->wc_esi, cpup->wc_ebp,
574 		    cpup->wc_esp))
575 		PMD(PMD_SX, ("cr0=%lx, cr3=%lx, cr4=%lx\n",
576 		    (long)cpup->wc_cr0, (long)cpup->wc_cr3,
577 		    (long)cpup->wc_cr4))
578 		PMD(PMD_SX, ("cs=%x, ds=%x, es=%x, ss=%x, fs=%lx, gs=%lx, "
579 		    "flgs=%lx\n", cpup->wc_cs, cpup->wc_ds, cpup->wc_es,
580 		    cpup->wc_ss, (long)cpup->wc_fs, (long)cpup->wc_gs,
581 		    (long)cpup->wc_eflags))
582 
583 		PMD(PMD_SX, ("gdt=%p:%x, idt=%p:%x, ldt=%lx, tr=%lx, "
584 		    "kgbase=%lx\n", (void *)cpup->wc_gdt_base,
585 		    cpup->wc_gdt_limit, (void *)cpup->wc_idt_base,
586 		    cpup->wc_idt_limit, (long)cpup->wc_ldt,
587 		    (long)cpup->wc_tr, (long)cpup->wc_kgsbase))
588 
589 		gdt.base = cpup->wc_gdt_base;
590 		gdt.limit = cpup->wc_gdt_limit;
591 
592 #if defined(__amd64)
593 		code_length = (uint32_t)wc_long_mode_64 -
594 		    (uint32_t)wc_rm_start;
595 #else
596 		code_length = 0;
597 #endif
598 
599 		init_real_mode_platter(0, code_length, cpup->wc_cr4, gdt);
600 
601 #if defined(__amd64)
602 		PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n",
603 		    (ulong_t)wcpp->rm_cr4, getcr4()))
604 
605 		PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n",
606 		    (ulong_t)wcpp->rm_pdbr, getcr3()))
607 
608 		PMD(PMD_SX, ("real_mode_platter->rm_longmode64_addr=%lx\n",
609 		    (ulong_t)wcpp->rm_longmode64_addr))
610 
611 		PMD(PMD_SX,
612 		    ("real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64]=%lx\n",
613 		    (ulong_t)wcpp->rm_temp_gdt[TEMPGDT_KCODE64]))
614 #endif
615 
616 		PMD(PMD_SX, ("gdt=%p:%x, idt=%p:%x, ldt=%lx, tr=%lx, "
617 		    "kgsbase=%lx\n", (void *)wcpp->rm_gdt_base,
618 		    wcpp->rm_gdt_lim, (void *)wcpp->rm_idt_base,
619 		    wcpp->rm_idt_lim, (long)cpup->wc_ldt, (long)cpup->wc_tr,
620 		    (long)cpup->wc_kgsbase))
621 
622 		power_req.request_type = PMR_PPM_ENTER_SX;
623 		power_req.req.ppm_power_enter_sx_req.sx_state = S3;
624 		power_req.req.ppm_power_enter_sx_req.test_point =
625 		    cpr_test_point;
626 		power_req.req.ppm_power_enter_sx_req.wakephys = wakephys;
627 
628 		PMD(PMD_SX, ("%s: pm_ctlops PMR_PPM_ENTER_SX\n", str))
629 		PT(PT_PPMCTLOP);
630 		(void) pm_ctlops(ppm, ddi_root_node(), DDI_CTLOPS_POWER,
631 		    &power_req, &ret);
632 		PMD(PMD_SX, ("%s: returns %d\n", str, ret))
633 
634 		/*
635 		 * If it works, we get control back to the else branch below
636 		 * If we get control back here, it didn't work.
637 		 * XXX return EINVAL here?
638 		 */
639 
640 		unmap_wakeaddr_1to1(wakephys);
641 		intr_restore(saved_intr);
642 
643 		return (ret);
644 	} else {
645 		cpr_suspend_succeeded = 1;
646 
647 		power_req.request_type = PMR_PPM_EXIT_SX;
648 		power_req.req.ppm_power_enter_sx_req.sx_state = S3;
649 
650 		PMD(PMD_SX, ("%s: pm_ctlops PMR_PPM_EXIT_SX\n", str))
651 		PT(PT_PPMCTLOP);
652 		(void) pm_ctlops(ppm, ddi_root_node(), DDI_CTLOPS_POWER,
653 		    &power_req, &ret);
654 		PMD(PMD_SX, ("%s: returns %d\n", str, ret))
655 
656 		ret = i_cpr_restore_apic(&(wc_other_cpus->wc_apic_state));
657 		/*
658 		 * the restore should never fail, if the saved suceeded
659 		 */
660 		ASSERT(ret == 0);
661 
662 		i_cpr_platform_free(&(wc_other_cpus->wc_apic_state));
663 
664 		PT(PT_INTRRESTORE);
665 		intr_restore(saved_intr);
666 		PT(PT_CPU);
667 
668 		return (ret);
669 	}
670 }
671 
672 /*
673  * Stop all other cpu's before halting or rebooting. We pause the cpu's
674  * instead of sending a cross call.
675  * Stolen from sun4/os/mp_states.c
676  */
677 
678 static int cpu_are_paused;	/* sic */
679 
680 void
681 i_cpr_stop_other_cpus(void)
682 {
683 	mutex_enter(&cpu_lock);
684 	if (cpu_are_paused) {
685 		mutex_exit(&cpu_lock);
686 		return;
687 	}
688 	pause_cpus(NULL);
689 	cpu_are_paused = 1;
690 
691 	mutex_exit(&cpu_lock);
692 }
693 
694 int
695 i_cpr_is_supported(int sleeptype)
696 {
697 	extern int cpr_supported_override;
698 	extern int cpr_platform_enable;
699 	extern int pm_S3_enabled;
700 
701 	if (sleeptype != CPR_TORAM)
702 		return (0);
703 
704 	/*
705 	 * The next statement tests if a specific platform has turned off
706 	 * cpr support.
707 	 */
708 	if (cpr_supported_override)
709 		return (0);
710 
711 	/*
712 	 * If a platform has specifically turned on cpr support ...
713 	 */
714 	if (cpr_platform_enable)
715 		return (1);
716 
717 	return (pm_S3_enabled);
718 }
719 
720 void
721 i_cpr_bitmap_cleanup(void)
722 {
723 }
724 
725 void
726 i_cpr_free_memory_resources(void)
727 {
728 }
729 
730 /*
731  * Needed only for S3 so far
732  */
733 static int
734 i_cpr_platform_alloc(psm_state_request_t *req)
735 {
736 	char	*str = "i_cpr_platform_alloc";
737 
738 	PMD(PMD_SX, ("cpu = %d, %s(%p) \n", CPU->cpu_id, str, (void *)req))
739 
740 	if (ncpus == 1) {
741 		PMD(PMD_SX, ("%s() : ncpus == 1\n", str))
742 		return (0);
743 	}
744 
745 	req->psr_cmd = PSM_STATE_ALLOC;
746 	return ((*psm_state)(req));
747 }
748 
749 /*
750  * Needed only for S3 so far
751  */
752 static void
753 i_cpr_platform_free(psm_state_request_t *req)
754 {
755 	char	*str = "i_cpr_platform_free";
756 
757 	PMD(PMD_SX, ("cpu = %d, %s(%p) \n", CPU->cpu_id, str, (void *)req))
758 
759 	if (ncpus == 1) {
760 		PMD(PMD_SX, ("%s() : ncpus == 1\n", str))
761 	}
762 
763 	req->psr_cmd = PSM_STATE_FREE;
764 	(void) (*psm_state)(req);
765 }
766 
767 static int
768 i_cpr_save_apic(psm_state_request_t *req)
769 {
770 	char	*str = "i_cpr_save_apic";
771 
772 	if (ncpus == 1) {
773 		PMD(PMD_SX, ("%s() : ncpus == 1\n", str))
774 		return (0);
775 	}
776 
777 	req->psr_cmd = PSM_STATE_SAVE;
778 	return ((*psm_state)(req));
779 }
780 
781 static int
782 i_cpr_restore_apic(psm_state_request_t *req)
783 {
784 	char	*str = "i_cpr_restore_apic";
785 
786 	if (ncpus == 1) {
787 		PMD(PMD_SX, ("%s() : ncpus == 1\n", str))
788 		return (0);
789 	}
790 
791 	req->psr_cmd = PSM_STATE_RESTORE;
792 	return ((*psm_state)(req));
793 }
794 
795 
796 /* stop lint complaining about offset not being used in 32bit mode */
797 #if !defined(__amd64)
798 /*ARGSUSED*/
799 #endif
800 static void
801 init_real_mode_platter(int cpun, uint32_t offset, uint_t cr4, wc_desctbr_t gdt)
802 {
803 	/*LINTED*/
804 	rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va;
805 
806 	/*
807 	 * Fill up the real mode platter to make it easy for real mode code to
808 	 * kick it off. This area should really be one passed by boot to kernel
809 	 * and guaranteed to be below 1MB and aligned to 16 bytes. Should also
810 	 * have identical physical and virtual address in paged mode.
811 	 */
812 
813 	real_mode_platter->rm_pdbr = getcr3();
814 	real_mode_platter->rm_cpu = cpun;
815 	real_mode_platter->rm_cr4 = cr4;
816 
817 	real_mode_platter->rm_gdt_base = gdt.base;
818 	real_mode_platter->rm_gdt_lim = gdt.limit;
819 
820 #if defined(__amd64)
821 	real_mode_platter->rm_x86feature = x86_feature;
822 
823 	if (getcr3() > 0xffffffffUL)
824 		panic("Cannot initialize CPUs; kernel's 64-bit page tables\n"
825 		    "located above 4G in physical memory (@ 0x%llx).",
826 		    (unsigned long long)getcr3());
827 
828 	/*
829 	 * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY
830 	 * by code in real_mode_start():
831 	 *
832 	 * GDT[0]:  NULL selector
833 	 * GDT[1]:  64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1
834 	 *
835 	 * Clear the IDT as interrupts will be off and a limit of 0 will cause
836 	 * the CPU to triple fault and reset on an NMI, seemingly as reasonable
837 	 * a course of action as any other, though it may cause the entire
838 	 * platform to reset in some cases...
839 	 */
840 	real_mode_platter->rm_temp_gdt[0] = 0ULL;
841 	real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL;
842 
843 	real_mode_platter->rm_temp_gdt_lim = (ushort_t)
844 	    (sizeof (real_mode_platter->rm_temp_gdt) - 1);
845 	real_mode_platter->rm_temp_gdt_base = rm_platter_pa +
846 	    (uint32_t)(&((rm_platter_t *)0)->rm_temp_gdt);
847 
848 	real_mode_platter->rm_temp_idt_lim = 0;
849 	real_mode_platter->rm_temp_idt_base = 0;
850 
851 	/*
852 	 * Since the CPU needs to jump to protected mode using an identity
853 	 * mapped address, we need to calculate it here.
854 	 */
855 	real_mode_platter->rm_longmode64_addr = rm_platter_pa + offset;
856 #endif	/* __amd64 */
857 
858 	/* return; */
859 }
860 
861 void
862 i_cpr_start_cpu(void)
863 {
864 
865 	struct cpu *cp = CPU;
866 
867 	char *str = "i_cpr_start_cpu";
868 	extern void init_cpu_syscall(struct cpu *cp);
869 
870 #if defined(__amd64)
871 	wc_cpu_t	*cpup = wc_other_cpus + cp->cpu_id;
872 #endif	/*	__amd64	*/
873 
874 	PMD(PMD_SX, ("%s() called\n", str))
875 
876 	PMD(PMD_SX, ("%s() #0 cp->cpu_base_spl %d\n", str,
877 	    cp->cpu_base_spl))
878 
879 	mutex_enter(&cpu_lock);
880 	if (cp == i_cpr_bootcpu()) {
881 		mutex_exit(&cpu_lock);
882 		PMD(PMD_SX,
883 		    ("%s() called on bootcpu nothing to do!\n", str))
884 		return;
885 	}
886 	mutex_exit(&cpu_lock);
887 
888 	/*
889 	 * We need to Sync PAT with cpu0's PAT. We have to do
890 	 * this with interrupts disabled.
891 	 */
892 	if (x86_feature & X86_PAT)
893 		pat_sync();
894 
895 	/*
896 	 * Initialize this CPU's syscall handlers
897 	 */
898 	init_cpu_syscall(cp);
899 
900 	PMD(PMD_SX, ("%s() #1 cp->cpu_base_spl %d\n", str, cp->cpu_base_spl))
901 
902 	/*
903 	 * Do not need to call cpuid_pass2(), cpuid_pass3(), cpuid_pass4() or
904 	 * init_cpu_info(), since the work that they do is only needed to
905 	 * be done once at boot time
906 	 */
907 
908 
909 	mutex_enter(&cpu_lock);
910 
911 #if defined(__amd64)
912 	restore_stack(cpup);
913 #endif	/*	__amd64	*/
914 
915 	CPUSET_ADD(procset, cp->cpu_id);
916 	mutex_exit(&cpu_lock);
917 
918 	PMD(PMD_SX, ("%s() #2 cp->cpu_base_spl %d\n", str,
919 	    cp->cpu_base_spl))
920 
921 	/* XXX remove before integration */
922 	PMD(PMD_SX, ("%s() procset 0x%lx\n", str, (ulong_t)procset))
923 
924 	if (tsc_gethrtime_enable) {
925 		PMD(PMD_SX, ("%s() calling tsc_sync_slave\n", str))
926 		tsc_sync_slave();
927 	}
928 
929 	PMD(PMD_SX, ("%s() cp->cpu_id %d, cp->cpu_intr_actv %d\n", str,
930 	    cp->cpu_id, cp->cpu_intr_actv))
931 	PMD(PMD_SX, ("%s() #3 cp->cpu_base_spl %d\n", str,
932 	    cp->cpu_base_spl))
933 
934 	(void) spl0();		/* enable interrupts */
935 
936 	PMD(PMD_SX, ("%s() #4 cp->cpu_base_spl %d\n", str,
937 	    cp->cpu_base_spl))
938 
939 	/*
940 	 * Set up the CPU module for this CPU.  This can't be done before
941 	 * this CPU is made CPU_READY, because we may (in heterogeneous systems)
942 	 * need to go load another CPU module.  The act of attempting to load
943 	 * a module may trigger a cross-call, which will ASSERT unless this
944 	 * cpu is CPU_READY.
945 	 */
946 
947 	/*
948 	 * cmi already been init'd (during boot), so do not need to do it again
949 	 */
950 #ifdef PM_REINITMCAONRESUME
951 	if (x86_feature & X86_MCA)
952 		cmi_mca_init();
953 #endif
954 
955 	PMD(PMD_SX, ("%s() returning\n", str))
956 
957 	/* return; */
958 }
959 
960 #if defined(__amd64)
961 /*
962  * we only need to do this for amd64!
963  */
964 
965 /*
966  * save the stack
967  */
968 void
969 save_stack(wc_cpu_t *cpup)
970 {
971 	char *str = "save_stack";
972 	caddr_t base = curthread->t_stk;
973 	caddr_t sp = (caddr_t)cpup->wc_rsp;
974 
975 
976 	PMD(PMD_SX, ("%s() CPU->cpu_id %d\n", str, CPU->cpu_id))
977 	PMD(PMD_SX, ("save_stack() curthread->t_stk = %p, sp = %p\n",
978 	    (void *)base, (void *)sp))
979 
980 	ASSERT(base > sp);
981 	/*LINTED*/
982 	bcopy(sp, cpup->wc_stack, base - sp);
983 
984 }
985 
986 /*
987  * restore the stack
988  */
989 static	void
990 restore_stack(wc_cpu_t *cpup)
991 {
992 	/*
993 	 * we only need to do this for amd64!
994 	 */
995 
996 	char *str = "restore_stack";
997 	caddr_t base = curthread->t_stk;
998 	caddr_t sp = (caddr_t)cpup->wc_rsp;
999 
1000 	PMD(PMD_SX, ("%s() CPU->cpu_id %d\n", str, CPU->cpu_id))
1001 	PMD(PMD_SX, ("%s() curthread->t_stk = %p, sp = %p\n", str,
1002 	    (void *)base, (void *)sp))
1003 
1004 	ASSERT(base > sp);
1005 	/*LINTED*/
1006 	bcopy(cpup->wc_stack, sp, base - sp);
1007 
1008 }
1009 
1010 #endif	/*	__amd64	*/
1011 
1012 
1013 void
1014 i_cpr_alloc_cpus(void)
1015 {
1016 	char *str = "i_cpr_alloc_cpus";
1017 
1018 	PMD(PMD_SX, ("%s() CPU->cpu_id %d\n", str, CPU->cpu_id))
1019 	/*
1020 	 * we allocate this only when we actually need it to save on
1021 	 * kernel memory
1022 	 */
1023 
1024 	if (wc_other_cpus == NULL) {
1025 		wc_other_cpus = kmem_zalloc(ncpus * sizeof (wc_cpu_t),
1026 		    KM_SLEEP);
1027 	}
1028 
1029 }
1030 
1031 void
1032 i_cpr_free_cpus(void)
1033 {
1034 	if (wc_other_cpus != NULL) {
1035 		kmem_free((void *) wc_other_cpus, ncpus * sizeof (wc_cpu_t));
1036 		wc_other_cpus = NULL;
1037 	}
1038 }
1039 
1040 /*
1041  * wrapper for acpica_ddi_save_resources()
1042  */
1043 void
1044 i_cpr_save_configuration(dev_info_t *dip)
1045 {
1046 	acpica_ddi_save_resources(dip);
1047 }
1048 
1049 /*
1050  * wrapper for acpica_ddi_restore_resources()
1051  */
1052 void
1053 i_cpr_restore_configuration(dev_info_t *dip)
1054 {
1055 	acpica_ddi_restore_resources(dip);
1056 }
1057 
1058 static int
1059 wait_for_set(cpuset_t *set, int who)
1060 {
1061 	int delays;
1062 	char *str = "wait_for_set";
1063 
1064 	for (delays = 0; !CPU_IN_SET(*set, who); delays++) {
1065 		if (delays == 500) {
1066 			/*
1067 			 * After five seconds, things are probably
1068 			 * looking a bit bleak - explain the hang.
1069 			 */
1070 			cmn_err(CE_NOTE, "cpu%d: started, "
1071 			    "but not running in the kernel yet", who);
1072 			PMD(PMD_SX, ("%s() %d cpu started "
1073 			    "but not running in the kernel yet\n",
1074 			    str, who))
1075 		} else if (delays > 2000) {
1076 			/*
1077 			 * We waited at least 20 seconds, bail ..
1078 			 */
1079 			cmn_err(CE_WARN, "cpu%d: timed out", who);
1080 			PMD(PMD_SX, ("%s() %d cpu timed out\n",
1081 			    str, who))
1082 			return (0);
1083 		}
1084 
1085 		/*
1086 		 * wait at least 10ms, then check again..
1087 		 */
1088 		drv_usecwait(10000);
1089 	}
1090 
1091 	return (1);
1092 }
1093