xref: /titanic_52/usr/src/uts/i86pc/io/pcplusmp/apic_common.c (revision f38cb554a534c6df738be3f4d23327e69888e634)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 /*
26  * Copyright (c) 2013, Joyent, Inc.  All rights reserved.
27  */
28 
29 /*
30  * PSMI 1.1 extensions are supported only in 2.6 and later versions.
31  * PSMI 1.2 extensions are supported only in 2.7 and later versions.
32  * PSMI 1.3 and 1.4 extensions are supported in Solaris 10.
33  * PSMI 1.5 extensions are supported in Solaris Nevada.
34  * PSMI 1.6 extensions are supported in Solaris Nevada.
35  * PSMI 1.7 extensions are supported in Solaris Nevada.
36  */
37 #define	PSMI_1_7
38 
39 #include <sys/processor.h>
40 #include <sys/time.h>
41 #include <sys/psm.h>
42 #include <sys/smp_impldefs.h>
43 #include <sys/cram.h>
44 #include <sys/acpi/acpi.h>
45 #include <sys/acpica.h>
46 #include <sys/psm_common.h>
47 #include <sys/apic.h>
48 #include <sys/pit.h>
49 #include <sys/ddi.h>
50 #include <sys/sunddi.h>
51 #include <sys/ddi_impldefs.h>
52 #include <sys/pci.h>
53 #include <sys/promif.h>
54 #include <sys/x86_archext.h>
55 #include <sys/cpc_impl.h>
56 #include <sys/uadmin.h>
57 #include <sys/panic.h>
58 #include <sys/debug.h>
59 #include <sys/archsystm.h>
60 #include <sys/trap.h>
61 #include <sys/machsystm.h>
62 #include <sys/sysmacros.h>
63 #include <sys/cpuvar.h>
64 #include <sys/rm_platter.h>
65 #include <sys/privregs.h>
66 #include <sys/note.h>
67 #include <sys/pci_intr_lib.h>
68 #include <sys/spl.h>
69 #include <sys/clock.h>
70 #include <sys/dditypes.h>
71 #include <sys/sunddi.h>
72 #include <sys/x_call.h>
73 #include <sys/reboot.h>
74 #include <sys/hpet.h>
75 #include <sys/apic_common.h>
76 #include <sys/apic_timer.h>
77 
78 static void	apic_record_ioapic_rdt(void *intrmap_private,
79 		    ioapic_rdt_t *irdt);
80 static void	apic_record_msi(void *intrmap_private, msi_regs_t *mregs);
81 
82 /*
83  * Common routines between pcplusmp & apix (taken from apic.c).
84  */
85 
86 int	apic_clkinit(int);
87 hrtime_t apic_gethrtime(void);
88 void	apic_send_ipi(int, int);
89 void	apic_set_idlecpu(processorid_t);
90 void	apic_unset_idlecpu(processorid_t);
91 void	apic_shutdown(int, int);
92 void	apic_preshutdown(int, int);
93 processorid_t	apic_get_next_processorid(processorid_t);
94 
95 hrtime_t apic_gettime();
96 
97 enum apic_ioapic_method_type apix_mul_ioapic_method = APIC_MUL_IOAPIC_PCPLUSMP;
98 
99 /* Now the ones for Dynamic Interrupt distribution */
100 int	apic_enable_dynamic_migration = 0;
101 
102 /* maximum loop count when sending Start IPIs. */
103 int apic_sipi_max_loop_count = 0x1000;
104 
105 /*
106  * These variables are frequently accessed in apic_intr_enter(),
107  * apic_intr_exit and apic_setspl, so group them together
108  */
109 volatile uint32_t *apicadr =  NULL;	/* virtual addr of local APIC	*/
110 int apic_setspl_delay = 1;		/* apic_setspl - delay enable	*/
111 int apic_clkvect;
112 
113 /* vector at which error interrupts come in */
114 int apic_errvect;
115 int apic_enable_error_intr = 1;
116 int apic_error_display_delay = 100;
117 
118 /* vector at which performance counter overflow interrupts come in */
119 int apic_cpcovf_vect;
120 int apic_enable_cpcovf_intr = 1;
121 
122 /* vector at which CMCI interrupts come in */
123 int apic_cmci_vect;
124 extern int cmi_enable_cmci;
125 extern void cmi_cmci_trap(void);
126 
127 kmutex_t cmci_cpu_setup_lock;	/* protects cmci_cpu_setup_registered */
128 int cmci_cpu_setup_registered;
129 
130 /* number of CPUs in power-on transition state */
131 static int apic_poweron_cnt = 0;
132 lock_t apic_mode_switch_lock;
133 
134 /*
135  * Patchable global variables.
136  */
137 int	apic_forceload = 0;
138 
139 int	apic_coarse_hrtime = 1;		/* 0 - use accurate slow gethrtime() */
140 
141 int	apic_flat_model = 0;		/* 0 - clustered. 1 - flat */
142 int	apic_panic_on_nmi = 0;
143 int	apic_panic_on_apic_error = 0;
144 
145 int	apic_verbose = 0;	/* 0x1ff */
146 
147 #ifdef DEBUG
148 int	apic_debug = 0;
149 int	apic_restrict_vector = 0;
150 
151 int	apic_debug_msgbuf[APIC_DEBUG_MSGBUFSIZE];
152 int	apic_debug_msgbufindex = 0;
153 
154 #endif /* DEBUG */
155 
156 uint_t apic_nticks = 0;
157 uint_t apic_skipped_redistribute = 0;
158 
159 uint_t last_count_read = 0;
160 lock_t	apic_gethrtime_lock;
161 volatile int	apic_hrtime_stamp = 0;
162 volatile hrtime_t apic_nsec_since_boot = 0;
163 
164 static	hrtime_t	apic_last_hrtime = 0;
165 int		apic_hrtime_error = 0;
166 int		apic_remote_hrterr = 0;
167 int		apic_num_nmis = 0;
168 int		apic_apic_error = 0;
169 int		apic_num_apic_errors = 0;
170 int		apic_num_cksum_errors = 0;
171 
172 int	apic_error = 0;
173 
174 static	int	apic_cmos_ssb_set = 0;
175 
176 /* use to make sure only one cpu handles the nmi */
177 lock_t	apic_nmi_lock;
178 /* use to make sure only one cpu handles the error interrupt */
179 lock_t	apic_error_lock;
180 
181 static	struct {
182 	uchar_t	cntl;
183 	uchar_t	data;
184 } aspen_bmc[] = {
185 	{ CC_SMS_WR_START,	0x18 },		/* NetFn/LUN */
186 	{ CC_SMS_WR_NEXT,	0x24 },		/* Cmd SET_WATCHDOG_TIMER */
187 	{ CC_SMS_WR_NEXT,	0x84 },		/* DataByte 1: SMS/OS no log */
188 	{ CC_SMS_WR_NEXT,	0x2 },		/* DataByte 2: Power Down */
189 	{ CC_SMS_WR_NEXT,	0x0 },		/* DataByte 3: no pre-timeout */
190 	{ CC_SMS_WR_NEXT,	0x0 },		/* DataByte 4: timer expir. */
191 	{ CC_SMS_WR_NEXT,	0xa },		/* DataByte 5: init countdown */
192 	{ CC_SMS_WR_END,	0x0 },		/* DataByte 6: init countdown */
193 
194 	{ CC_SMS_WR_START,	0x18 },		/* NetFn/LUN */
195 	{ CC_SMS_WR_END,	0x22 }		/* Cmd RESET_WATCHDOG_TIMER */
196 };
197 
198 static	struct {
199 	int	port;
200 	uchar_t	data;
201 } sitka_bmc[] = {
202 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_START },
203 	{ SMS_DATA_REGISTER,	0x18 },		/* NetFn/LUN */
204 	{ SMS_DATA_REGISTER,	0x24 },		/* Cmd SET_WATCHDOG_TIMER */
205 	{ SMS_DATA_REGISTER,	0x84 },		/* DataByte 1: SMS/OS no log */
206 	{ SMS_DATA_REGISTER,	0x2 },		/* DataByte 2: Power Down */
207 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 3: no pre-timeout */
208 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 4: timer expir. */
209 	{ SMS_DATA_REGISTER,	0xa },		/* DataByte 5: init countdown */
210 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_END },
211 	{ SMS_DATA_REGISTER,	0x0 },		/* DataByte 6: init countdown */
212 
213 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_START },
214 	{ SMS_DATA_REGISTER,	0x18 },		/* NetFn/LUN */
215 	{ SMS_COMMAND_REGISTER,	SMS_WRITE_END },
216 	{ SMS_DATA_REGISTER,	0x22 }		/* Cmd RESET_WATCHDOG_TIMER */
217 };
218 
219 /* Patchable global variables. */
220 int		apic_kmdb_on_nmi = 0;		/* 0 - no, 1 - yes enter kmdb */
221 uint32_t	apic_divide_reg_init = 0;	/* 0 - divide by 2 */
222 
223 /* default apic ops without interrupt remapping */
224 static apic_intrmap_ops_t apic_nointrmap_ops = {
225 	(int (*)(int))return_instr,
226 	(void (*)(int))return_instr,
227 	(void (*)(void **, dev_info_t *, uint16_t, int, uchar_t))return_instr,
228 	(void (*)(void *, void *, uint16_t, int))return_instr,
229 	(void (*)(void **))return_instr,
230 	apic_record_ioapic_rdt,
231 	apic_record_msi,
232 };
233 
234 apic_intrmap_ops_t *apic_vt_ops = &apic_nointrmap_ops;
235 apic_cpus_info_t	*apic_cpus = NULL;
236 cpuset_t	apic_cpumask;
237 uint_t		apic_picinit_called;
238 
239 /* Flag to indicate that we need to shut down all processors */
240 static uint_t	apic_shutdown_processors;
241 
242 /*
243  * Probe the ioapic method for apix module. Called in apic_probe_common()
244  */
245 int
246 apic_ioapic_method_probe()
247 {
248 	if (apix_enable == 0)
249 		return (PSM_SUCCESS);
250 
251 	/*
252 	 * Set IOAPIC EOI handling method. The priority from low to high is:
253 	 * 	1. IOxAPIC: with EOI register
254 	 * 	2. IOMMU interrupt mapping
255 	 *	3. Mask-Before-EOI method for systems without boot
256 	 *	interrupt routing, such as systems with only one IOAPIC;
257 	 *	NVIDIA CK8-04/MCP55 systems; systems with bridge solution
258 	 *	which disables the boot interrupt routing already.
259 	 * 	4. Directed EOI
260 	 */
261 	if (apic_io_ver[0] >= 0x20)
262 		apix_mul_ioapic_method = APIC_MUL_IOAPIC_IOXAPIC;
263 	if ((apic_io_max == 1) || (apic_nvidia_io_max == apic_io_max))
264 		apix_mul_ioapic_method = APIC_MUL_IOAPIC_MASK;
265 	if (apic_directed_EOI_supported())
266 		apix_mul_ioapic_method = APIC_MUL_IOAPIC_DEOI;
267 
268 	/* fall back to pcplusmp */
269 	if (apix_mul_ioapic_method == APIC_MUL_IOAPIC_PCPLUSMP) {
270 		/* make sure apix is after pcplusmp in /etc/mach */
271 		apix_enable = 0; /* go ahead with pcplusmp install next */
272 		return (PSM_FAILURE);
273 	}
274 
275 	return (PSM_SUCCESS);
276 }
277 
278 /*
279  * handler for APIC Error interrupt. Just print a warning and continue
280  */
281 int
282 apic_error_intr()
283 {
284 	uint_t	error0, error1, error;
285 	uint_t	i;
286 
287 	/*
288 	 * We need to write before read as per 7.4.17 of system prog manual.
289 	 * We do both and or the results to be safe
290 	 */
291 	error0 = apic_reg_ops->apic_read(APIC_ERROR_STATUS);
292 	apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
293 	error1 = apic_reg_ops->apic_read(APIC_ERROR_STATUS);
294 	error = error0 | error1;
295 
296 	/*
297 	 * Clear the APIC error status (do this on all cpus that enter here)
298 	 * (two writes are required due to the semantics of accessing the
299 	 * error status register.)
300 	 */
301 	apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
302 	apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0);
303 
304 	/*
305 	 * Prevent more than 1 CPU from handling error interrupt causing
306 	 * double printing (interleave of characters from multiple
307 	 * CPU's when using prom_printf)
308 	 */
309 	if (lock_try(&apic_error_lock) == 0)
310 		return (error ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
311 	if (error) {
312 #if	DEBUG
313 		if (apic_debug)
314 			debug_enter("pcplusmp: APIC Error interrupt received");
315 #endif /* DEBUG */
316 		if (apic_panic_on_apic_error)
317 			cmn_err(CE_PANIC,
318 			    "APIC Error interrupt on CPU %d. Status = %x",
319 			    psm_get_cpu_id(), error);
320 		else {
321 			if ((error & ~APIC_CS_ERRORS) == 0) {
322 				/* cksum error only */
323 				apic_error |= APIC_ERR_APIC_ERROR;
324 				apic_apic_error |= error;
325 				apic_num_apic_errors++;
326 				apic_num_cksum_errors++;
327 			} else {
328 				/*
329 				 * prom_printf is the best shot we have of
330 				 * something which is problem free from
331 				 * high level/NMI type of interrupts
332 				 */
333 				prom_printf("APIC Error interrupt on CPU %d. "
334 				    "Status 0 = %x, Status 1 = %x\n",
335 				    psm_get_cpu_id(), error0, error1);
336 				apic_error |= APIC_ERR_APIC_ERROR;
337 				apic_apic_error |= error;
338 				apic_num_apic_errors++;
339 				for (i = 0; i < apic_error_display_delay; i++) {
340 					tenmicrosec();
341 				}
342 				/*
343 				 * provide more delay next time limited to
344 				 * roughly 1 clock tick time
345 				 */
346 				if (apic_error_display_delay < 500)
347 					apic_error_display_delay *= 2;
348 			}
349 		}
350 		lock_clear(&apic_error_lock);
351 		return (DDI_INTR_CLAIMED);
352 	} else {
353 		lock_clear(&apic_error_lock);
354 		return (DDI_INTR_UNCLAIMED);
355 	}
356 }
357 
358 /*
359  * Turn off the mask bit in the performance counter Local Vector Table entry.
360  */
361 void
362 apic_cpcovf_mask_clear(void)
363 {
364 	apic_reg_ops->apic_write(APIC_PCINT_VECT,
365 	    (apic_reg_ops->apic_read(APIC_PCINT_VECT) & ~APIC_LVT_MASK));
366 }
367 
368 /*ARGSUSED*/
369 static int
370 apic_cmci_enable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3)
371 {
372 	apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect);
373 	return (0);
374 }
375 
376 /*ARGSUSED*/
377 static int
378 apic_cmci_disable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3)
379 {
380 	apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect | AV_MASK);
381 	return (0);
382 }
383 
384 /*ARGSUSED*/
385 int
386 cmci_cpu_setup(cpu_setup_t what, int cpuid, void *arg)
387 {
388 	cpuset_t	cpu_set;
389 
390 	CPUSET_ONLY(cpu_set, cpuid);
391 
392 	switch (what) {
393 		case CPU_ON:
394 			xc_call(NULL, NULL, NULL, CPUSET2BV(cpu_set),
395 			    (xc_func_t)apic_cmci_enable);
396 			break;
397 
398 		case CPU_OFF:
399 			xc_call(NULL, NULL, NULL, CPUSET2BV(cpu_set),
400 			    (xc_func_t)apic_cmci_disable);
401 			break;
402 
403 		default:
404 			break;
405 	}
406 
407 	return (0);
408 }
409 
410 static void
411 apic_disable_local_apic(void)
412 {
413 	apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL);
414 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK);
415 
416 	/* local intr reg 0 */
417 	apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK);
418 
419 	/* disable NMI */
420 	apic_reg_ops->apic_write(APIC_INT_VECT1, AV_MASK);
421 
422 	/* and error interrupt */
423 	apic_reg_ops->apic_write(APIC_ERR_VECT, AV_MASK);
424 
425 	/* and perf counter intr */
426 	apic_reg_ops->apic_write(APIC_PCINT_VECT, AV_MASK);
427 
428 	apic_reg_ops->apic_write(APIC_SPUR_INT_REG, APIC_SPUR_INTR);
429 }
430 
431 static void
432 apic_cpu_send_SIPI(processorid_t cpun, boolean_t start)
433 {
434 	int		loop_count;
435 	uint32_t	vector;
436 	uint_t		apicid;
437 	ulong_t		iflag;
438 
439 	apicid =  apic_cpus[cpun].aci_local_id;
440 
441 	/*
442 	 * Interrupts on current CPU will be disabled during the
443 	 * steps in order to avoid unwanted side effects from
444 	 * executing interrupt handlers on a problematic BIOS.
445 	 */
446 	iflag = intr_clear();
447 
448 	if (start) {
449 		outb(CMOS_ADDR, SSB);
450 		outb(CMOS_DATA, BIOS_SHUTDOWN);
451 	}
452 
453 	/*
454 	 * According to X2APIC specification in section '2.3.5.1' of
455 	 * Interrupt Command Register Semantics, the semantics of
456 	 * programming the Interrupt Command Register to dispatch an interrupt
457 	 * is simplified. A single MSR write to the 64-bit ICR is required
458 	 * for dispatching an interrupt. Specifically, with the 64-bit MSR
459 	 * interface to ICR, system software is not required to check the
460 	 * status of the delivery status bit prior to writing to the ICR
461 	 * to send an IPI. With the removal of the Delivery Status bit,
462 	 * system software no longer has a reason to read the ICR. It remains
463 	 * readable only to aid in debugging.
464 	 */
465 #ifdef	DEBUG
466 	APIC_AV_PENDING_SET();
467 #else
468 	if (apic_mode == LOCAL_APIC) {
469 		APIC_AV_PENDING_SET();
470 	}
471 #endif /* DEBUG */
472 
473 	/* for integrated - make sure there is one INIT IPI in buffer */
474 	/* for external - it will wake up the cpu */
475 	apic_reg_ops->apic_write_int_cmd(apicid, AV_ASSERT | AV_RESET);
476 
477 	/* If only 1 CPU is installed, PENDING bit will not go low */
478 	for (loop_count = apic_sipi_max_loop_count; loop_count; loop_count--) {
479 		if (apic_mode == LOCAL_APIC &&
480 		    apic_reg_ops->apic_read(APIC_INT_CMD1) & AV_PENDING)
481 			apic_ret();
482 		else
483 			break;
484 	}
485 
486 	apic_reg_ops->apic_write_int_cmd(apicid, AV_DEASSERT | AV_RESET);
487 	drv_usecwait(20000);		/* 20 milli sec */
488 
489 	if (apic_cpus[cpun].aci_local_ver >= APIC_INTEGRATED_VERS) {
490 		/* integrated apic */
491 
492 		vector = (rm_platter_pa >> MMU_PAGESHIFT) &
493 		    (APIC_VECTOR_MASK | APIC_IPL_MASK);
494 
495 		/* to offset the INIT IPI queue up in the buffer */
496 		apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP);
497 		drv_usecwait(200);		/* 20 micro sec */
498 
499 		/*
500 		 * send the second SIPI (Startup IPI) as recommended by Intel
501 		 * software development manual.
502 		 */
503 		apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP);
504 		drv_usecwait(200);	/* 20 micro sec */
505 	}
506 
507 	intr_restore(iflag);
508 }
509 
510 /*ARGSUSED1*/
511 int
512 apic_cpu_start(processorid_t cpun, caddr_t arg)
513 {
514 	ASSERT(MUTEX_HELD(&cpu_lock));
515 
516 	if (!apic_cpu_in_range(cpun)) {
517 		return (EINVAL);
518 	}
519 
520 	/*
521 	 * Switch to apic_common_send_ipi for safety during starting other CPUs.
522 	 */
523 	if (apic_mode == LOCAL_X2APIC) {
524 		apic_switch_ipi_callback(B_TRUE);
525 	}
526 
527 	apic_cmos_ssb_set = 1;
528 	apic_cpu_send_SIPI(cpun, B_TRUE);
529 
530 	return (0);
531 }
532 
533 /*
534  * Put CPU into halted state with interrupts disabled.
535  */
536 /*ARGSUSED1*/
537 int
538 apic_cpu_stop(processorid_t cpun, caddr_t arg)
539 {
540 	int		rc;
541 	cpu_t 		*cp;
542 	extern cpuset_t cpu_ready_set;
543 	extern void cpu_idle_intercept_cpu(cpu_t *cp);
544 
545 	ASSERT(MUTEX_HELD(&cpu_lock));
546 
547 	if (!apic_cpu_in_range(cpun)) {
548 		return (EINVAL);
549 	}
550 	if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) {
551 		return (ENOTSUP);
552 	}
553 
554 	cp = cpu_get(cpun);
555 	ASSERT(cp != NULL);
556 	ASSERT((cp->cpu_flags & CPU_OFFLINE) != 0);
557 	ASSERT((cp->cpu_flags & CPU_QUIESCED) != 0);
558 	ASSERT((cp->cpu_flags & CPU_ENABLE) == 0);
559 
560 	/* Clear CPU_READY flag to disable cross calls. */
561 	cp->cpu_flags &= ~CPU_READY;
562 	CPUSET_ATOMIC_DEL(cpu_ready_set, cpun);
563 	rc = xc_flush_cpu(cp);
564 	if (rc != 0) {
565 		CPUSET_ATOMIC_ADD(cpu_ready_set, cpun);
566 		cp->cpu_flags |= CPU_READY;
567 		return (rc);
568 	}
569 
570 	/* Intercept target CPU at a safe point before powering it off. */
571 	cpu_idle_intercept_cpu(cp);
572 
573 	apic_cpu_send_SIPI(cpun, B_FALSE);
574 	cp->cpu_flags &= ~CPU_RUNNING;
575 
576 	return (0);
577 }
578 
579 int
580 apic_cpu_ops(psm_cpu_request_t *reqp)
581 {
582 	if (reqp == NULL) {
583 		return (EINVAL);
584 	}
585 
586 	switch (reqp->pcr_cmd) {
587 	case PSM_CPU_ADD:
588 		return (apic_cpu_add(reqp));
589 
590 	case PSM_CPU_REMOVE:
591 		return (apic_cpu_remove(reqp));
592 
593 	case PSM_CPU_STOP:
594 		return (apic_cpu_stop(reqp->req.cpu_stop.cpuid,
595 		    reqp->req.cpu_stop.ctx));
596 
597 	default:
598 		return (ENOTSUP);
599 	}
600 }
601 
602 #ifdef	DEBUG
603 int	apic_break_on_cpu = 9;
604 int	apic_stretch_interrupts = 0;
605 int	apic_stretch_ISR = 1 << 3;	/* IPL of 3 matches nothing now */
606 #endif /* DEBUG */
607 
608 /*
609  * generates an interprocessor interrupt to another CPU. Any changes made to
610  * this routine must be accompanied by similar changes to
611  * apic_common_send_ipi().
612  */
613 void
614 apic_send_ipi(int cpun, int ipl)
615 {
616 	int vector;
617 	ulong_t flag;
618 
619 	vector = apic_resv_vector[ipl];
620 
621 	ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
622 
623 	flag = intr_clear();
624 
625 	APIC_AV_PENDING_SET();
626 
627 	apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
628 	    vector);
629 
630 	intr_restore(flag);
631 }
632 
633 
634 /*ARGSUSED*/
635 void
636 apic_set_idlecpu(processorid_t cpun)
637 {
638 }
639 
640 /*ARGSUSED*/
641 void
642 apic_unset_idlecpu(processorid_t cpun)
643 {
644 }
645 
646 
647 void
648 apic_ret()
649 {
650 }
651 
652 /*
653  * If apic_coarse_time == 1, then apic_gettime() is used instead of
654  * apic_gethrtime().  This is used for performance instead of accuracy.
655  */
656 
657 hrtime_t
658 apic_gettime()
659 {
660 	int old_hrtime_stamp;
661 	hrtime_t temp;
662 
663 	/*
664 	 * In one-shot mode, we do not keep time, so if anyone
665 	 * calls psm_gettime() directly, we vector over to
666 	 * gethrtime().
667 	 * one-shot mode MUST NOT be enabled if this psm is the source of
668 	 * hrtime.
669 	 */
670 
671 	if (apic_oneshot)
672 		return (gethrtime());
673 
674 
675 gettime_again:
676 	while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
677 		apic_ret();
678 
679 	temp = apic_nsec_since_boot;
680 
681 	if (apic_hrtime_stamp != old_hrtime_stamp) {	/* got an interrupt */
682 		goto gettime_again;
683 	}
684 	return (temp);
685 }
686 
687 /*
688  * Here we return the number of nanoseconds since booting.  Note every
689  * clock interrupt increments apic_nsec_since_boot by the appropriate
690  * amount.
691  */
692 hrtime_t
693 apic_gethrtime(void)
694 {
695 	int curr_timeval, countval, elapsed_ticks;
696 	int old_hrtime_stamp, status;
697 	hrtime_t temp;
698 	uint32_t cpun;
699 	ulong_t oflags;
700 
701 	/*
702 	 * In one-shot mode, we do not keep time, so if anyone
703 	 * calls psm_gethrtime() directly, we vector over to
704 	 * gethrtime().
705 	 * one-shot mode MUST NOT be enabled if this psm is the source of
706 	 * hrtime.
707 	 */
708 
709 	if (apic_oneshot)
710 		return (gethrtime());
711 
712 	oflags = intr_clear();	/* prevent migration */
713 
714 	cpun = apic_reg_ops->apic_read(APIC_LID_REG);
715 	if (apic_mode == LOCAL_APIC)
716 		cpun >>= APIC_ID_BIT_OFFSET;
717 
718 	lock_set(&apic_gethrtime_lock);
719 
720 gethrtime_again:
721 	while ((old_hrtime_stamp = apic_hrtime_stamp) & 1)
722 		apic_ret();
723 
724 	/*
725 	 * Check to see which CPU we are on.  Note the time is kept on
726 	 * the local APIC of CPU 0.  If on CPU 0, simply read the current
727 	 * counter.  If on another CPU, issue a remote read command to CPU 0.
728 	 */
729 	if (cpun == apic_cpus[0].aci_local_id) {
730 		countval = apic_reg_ops->apic_read(APIC_CURR_COUNT);
731 	} else {
732 #ifdef	DEBUG
733 		APIC_AV_PENDING_SET();
734 #else
735 		if (apic_mode == LOCAL_APIC)
736 			APIC_AV_PENDING_SET();
737 #endif /* DEBUG */
738 
739 		apic_reg_ops->apic_write_int_cmd(
740 		    apic_cpus[0].aci_local_id, APIC_CURR_ADD | AV_REMOTE);
741 
742 		while ((status = apic_reg_ops->apic_read(APIC_INT_CMD1))
743 		    & AV_READ_PENDING) {
744 			apic_ret();
745 		}
746 
747 		if (status & AV_REMOTE_STATUS)	/* 1 = valid */
748 			countval = apic_reg_ops->apic_read(APIC_REMOTE_READ);
749 		else {	/* 0 = invalid */
750 			apic_remote_hrterr++;
751 			/*
752 			 * return last hrtime right now, will need more
753 			 * testing if change to retry
754 			 */
755 			temp = apic_last_hrtime;
756 
757 			lock_clear(&apic_gethrtime_lock);
758 
759 			intr_restore(oflags);
760 
761 			return (temp);
762 		}
763 	}
764 	if (countval > last_count_read)
765 		countval = 0;
766 	else
767 		last_count_read = countval;
768 
769 	elapsed_ticks = apic_hertz_count - countval;
770 
771 	curr_timeval = APIC_TICKS_TO_NSECS(elapsed_ticks);
772 	temp = apic_nsec_since_boot + curr_timeval;
773 
774 	if (apic_hrtime_stamp != old_hrtime_stamp) {	/* got an interrupt */
775 		/* we might have clobbered last_count_read. Restore it */
776 		last_count_read = apic_hertz_count;
777 		goto gethrtime_again;
778 	}
779 
780 	if (temp < apic_last_hrtime) {
781 		/* return last hrtime if error occurs */
782 		apic_hrtime_error++;
783 		temp = apic_last_hrtime;
784 	}
785 	else
786 		apic_last_hrtime = temp;
787 
788 	lock_clear(&apic_gethrtime_lock);
789 	intr_restore(oflags);
790 
791 	return (temp);
792 }
793 
794 /* apic NMI handler */
795 /*ARGSUSED*/
796 void
797 apic_nmi_intr(caddr_t arg, struct regs *rp)
798 {
799 	if (apic_shutdown_processors) {
800 		apic_disable_local_apic();
801 		return;
802 	}
803 
804 	apic_error |= APIC_ERR_NMI;
805 
806 	if (!lock_try(&apic_nmi_lock))
807 		return;
808 	apic_num_nmis++;
809 
810 	if (apic_kmdb_on_nmi && psm_debugger()) {
811 		debug_enter("NMI received: entering kmdb\n");
812 	} else if (apic_panic_on_nmi) {
813 		/* Keep panic from entering kmdb. */
814 		nopanicdebug = 1;
815 		panic("NMI received\n");
816 	} else {
817 		/*
818 		 * prom_printf is the best shot we have of something which is
819 		 * problem free from high level/NMI type of interrupts
820 		 */
821 		prom_printf("NMI received\n");
822 	}
823 
824 	lock_clear(&apic_nmi_lock);
825 }
826 
827 processorid_t
828 apic_get_next_processorid(processorid_t cpu_id)
829 {
830 
831 	int i;
832 
833 	if (cpu_id == -1)
834 		return ((processorid_t)0);
835 
836 	for (i = cpu_id + 1; i < NCPU; i++) {
837 		if (apic_cpu_in_range(i))
838 			return (i);
839 	}
840 
841 	return ((processorid_t)-1);
842 }
843 
844 int
845 apic_cpu_add(psm_cpu_request_t *reqp)
846 {
847 	int i, rv = 0;
848 	ulong_t iflag;
849 	boolean_t first = B_TRUE;
850 	uchar_t localver;
851 	uint32_t localid, procid;
852 	processorid_t cpuid = (processorid_t)-1;
853 	mach_cpu_add_arg_t *ap;
854 
855 	ASSERT(reqp != NULL);
856 	reqp->req.cpu_add.cpuid = (processorid_t)-1;
857 
858 	/* Check whether CPU hotplug is supported. */
859 	if (!plat_dr_support_cpu() || apic_max_nproc == -1) {
860 		return (ENOTSUP);
861 	}
862 
863 	ap = (mach_cpu_add_arg_t *)reqp->req.cpu_add.argp;
864 	switch (ap->type) {
865 	case MACH_CPU_ARG_LOCAL_APIC:
866 		localid = ap->arg.apic.apic_id;
867 		procid = ap->arg.apic.proc_id;
868 		if (localid >= 255 || procid > 255) {
869 			cmn_err(CE_WARN,
870 			    "!apic: apicid(%u) or procid(%u) is invalid.",
871 			    localid, procid);
872 			return (EINVAL);
873 		}
874 		break;
875 
876 	case MACH_CPU_ARG_LOCAL_X2APIC:
877 		localid = ap->arg.apic.apic_id;
878 		procid = ap->arg.apic.proc_id;
879 		if (localid >= UINT32_MAX) {
880 			cmn_err(CE_WARN,
881 			    "!apic: x2apicid(%u) is invalid.", localid);
882 			return (EINVAL);
883 		} else if (localid >= 255 && apic_mode == LOCAL_APIC) {
884 			cmn_err(CE_WARN, "!apic: system is in APIC mode, "
885 			    "can't support x2APIC processor.");
886 			return (ENOTSUP);
887 		}
888 		break;
889 
890 	default:
891 		cmn_err(CE_WARN,
892 		    "!apic: unknown argument type %d to apic_cpu_add().",
893 		    ap->type);
894 		return (EINVAL);
895 	}
896 
897 	/* Use apic_ioapic_lock to sync with apic_get_next_bind_cpu. */
898 	iflag = intr_clear();
899 	lock_set(&apic_ioapic_lock);
900 
901 	/* Check whether local APIC id already exists. */
902 	for (i = 0; i < apic_nproc; i++) {
903 		if (!CPU_IN_SET(apic_cpumask, i))
904 			continue;
905 		if (apic_cpus[i].aci_local_id == localid) {
906 			lock_clear(&apic_ioapic_lock);
907 			intr_restore(iflag);
908 			cmn_err(CE_WARN,
909 			    "!apic: local apic id %u already exists.",
910 			    localid);
911 			return (EEXIST);
912 		} else if (apic_cpus[i].aci_processor_id == procid) {
913 			lock_clear(&apic_ioapic_lock);
914 			intr_restore(iflag);
915 			cmn_err(CE_WARN,
916 			    "!apic: processor id %u already exists.",
917 			    (int)procid);
918 			return (EEXIST);
919 		}
920 
921 		/*
922 		 * There's no local APIC version number available in MADT table,
923 		 * so assume that all CPUs are homogeneous and use local APIC
924 		 * version number of the first existing CPU.
925 		 */
926 		if (first) {
927 			first = B_FALSE;
928 			localver = apic_cpus[i].aci_local_ver;
929 		}
930 	}
931 	ASSERT(first == B_FALSE);
932 
933 	/*
934 	 * Try to assign the same cpuid if APIC id exists in the dirty cache.
935 	 */
936 	for (i = 0; i < apic_max_nproc; i++) {
937 		if (CPU_IN_SET(apic_cpumask, i)) {
938 			ASSERT((apic_cpus[i].aci_status & APIC_CPU_FREE) == 0);
939 			continue;
940 		}
941 		ASSERT(apic_cpus[i].aci_status & APIC_CPU_FREE);
942 		if ((apic_cpus[i].aci_status & APIC_CPU_DIRTY) &&
943 		    apic_cpus[i].aci_local_id == localid &&
944 		    apic_cpus[i].aci_processor_id == procid) {
945 			cpuid = i;
946 			break;
947 		}
948 	}
949 
950 	/* Avoid the dirty cache and allocate fresh slot if possible. */
951 	if (cpuid == (processorid_t)-1) {
952 		for (i = 0; i < apic_max_nproc; i++) {
953 			if ((apic_cpus[i].aci_status & APIC_CPU_FREE) &&
954 			    (apic_cpus[i].aci_status & APIC_CPU_DIRTY) == 0) {
955 				cpuid = i;
956 				break;
957 			}
958 		}
959 	}
960 
961 	/* Try to find any free slot as last resort. */
962 	if (cpuid == (processorid_t)-1) {
963 		for (i = 0; i < apic_max_nproc; i++) {
964 			if (apic_cpus[i].aci_status & APIC_CPU_FREE) {
965 				cpuid = i;
966 				break;
967 			}
968 		}
969 	}
970 
971 	if (cpuid == (processorid_t)-1) {
972 		lock_clear(&apic_ioapic_lock);
973 		intr_restore(iflag);
974 		cmn_err(CE_NOTE,
975 		    "!apic: failed to allocate cpu id for processor %u.",
976 		    procid);
977 		rv = EAGAIN;
978 	} else if (ACPI_FAILURE(acpica_map_cpu(cpuid, procid))) {
979 		lock_clear(&apic_ioapic_lock);
980 		intr_restore(iflag);
981 		cmn_err(CE_NOTE,
982 		    "!apic: failed to build mapping for processor %u.",
983 		    procid);
984 		rv = EBUSY;
985 	} else {
986 		ASSERT(cpuid >= 0 && cpuid < NCPU);
987 		ASSERT(cpuid < apic_max_nproc && cpuid < max_ncpus);
988 		bzero(&apic_cpus[cpuid], sizeof (apic_cpus[0]));
989 		apic_cpus[cpuid].aci_processor_id = procid;
990 		apic_cpus[cpuid].aci_local_id = localid;
991 		apic_cpus[cpuid].aci_local_ver = localver;
992 		CPUSET_ATOMIC_ADD(apic_cpumask, cpuid);
993 		if (cpuid >= apic_nproc) {
994 			apic_nproc = cpuid + 1;
995 		}
996 		lock_clear(&apic_ioapic_lock);
997 		intr_restore(iflag);
998 		reqp->req.cpu_add.cpuid = cpuid;
999 	}
1000 
1001 	return (rv);
1002 }
1003 
1004 int
1005 apic_cpu_remove(psm_cpu_request_t *reqp)
1006 {
1007 	int i;
1008 	ulong_t iflag;
1009 	processorid_t cpuid;
1010 
1011 	/* Check whether CPU hotplug is supported. */
1012 	if (!plat_dr_support_cpu() || apic_max_nproc == -1) {
1013 		return (ENOTSUP);
1014 	}
1015 
1016 	cpuid = reqp->req.cpu_remove.cpuid;
1017 
1018 	/* Use apic_ioapic_lock to sync with apic_get_next_bind_cpu. */
1019 	iflag = intr_clear();
1020 	lock_set(&apic_ioapic_lock);
1021 
1022 	if (!apic_cpu_in_range(cpuid)) {
1023 		lock_clear(&apic_ioapic_lock);
1024 		intr_restore(iflag);
1025 		cmn_err(CE_WARN,
1026 		    "!apic: cpuid %d doesn't exist in apic_cpus array.",
1027 		    cpuid);
1028 		return (ENODEV);
1029 	}
1030 	ASSERT((apic_cpus[cpuid].aci_status & APIC_CPU_FREE) == 0);
1031 
1032 	if (ACPI_FAILURE(acpica_unmap_cpu(cpuid))) {
1033 		lock_clear(&apic_ioapic_lock);
1034 		intr_restore(iflag);
1035 		return (ENOENT);
1036 	}
1037 
1038 	if (cpuid == apic_nproc - 1) {
1039 		/*
1040 		 * We are removing the highest numbered cpuid so we need to
1041 		 * find the next highest cpuid as the new value for apic_nproc.
1042 		 */
1043 		for (i = apic_nproc; i > 0; i--) {
1044 			if (CPU_IN_SET(apic_cpumask, i - 1)) {
1045 				apic_nproc = i;
1046 				break;
1047 			}
1048 		}
1049 		/* at least one CPU left */
1050 		ASSERT(i > 0);
1051 	}
1052 	CPUSET_ATOMIC_DEL(apic_cpumask, cpuid);
1053 	/* mark slot as free and keep it in the dirty cache */
1054 	apic_cpus[cpuid].aci_status = APIC_CPU_FREE | APIC_CPU_DIRTY;
1055 
1056 	lock_clear(&apic_ioapic_lock);
1057 	intr_restore(iflag);
1058 
1059 	return (0);
1060 }
1061 
1062 /*
1063  * Return the number of APIC clock ticks elapsed for 8245 to decrement
1064  * (APIC_TIME_COUNT + pit_ticks_adj) ticks.
1065  */
1066 uint_t
1067 apic_calibrate(volatile uint32_t *addr, uint16_t *pit_ticks_adj)
1068 {
1069 	uint8_t		pit_tick_lo;
1070 	uint16_t	pit_tick, target_pit_tick;
1071 	uint32_t	start_apic_tick, end_apic_tick;
1072 	ulong_t		iflag;
1073 	uint32_t	reg;
1074 
1075 	reg = addr + APIC_CURR_COUNT - apicadr;
1076 
1077 	iflag = intr_clear();
1078 
1079 	do {
1080 		pit_tick_lo = inb(PITCTR0_PORT);
1081 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1082 	} while (pit_tick < APIC_TIME_MIN ||
1083 	    pit_tick_lo <= APIC_LB_MIN || pit_tick_lo >= APIC_LB_MAX);
1084 
1085 	/*
1086 	 * Wait for the 8254 to decrement by 5 ticks to ensure
1087 	 * we didn't start in the middle of a tick.
1088 	 * Compare with 0x10 for the wrap around case.
1089 	 */
1090 	target_pit_tick = pit_tick - 5;
1091 	do {
1092 		pit_tick_lo = inb(PITCTR0_PORT);
1093 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1094 	} while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1095 
1096 	start_apic_tick = apic_reg_ops->apic_read(reg);
1097 
1098 	/*
1099 	 * Wait for the 8254 to decrement by
1100 	 * (APIC_TIME_COUNT + pit_ticks_adj) ticks
1101 	 */
1102 	target_pit_tick = pit_tick - APIC_TIME_COUNT;
1103 	do {
1104 		pit_tick_lo = inb(PITCTR0_PORT);
1105 		pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo;
1106 	} while (pit_tick > target_pit_tick || pit_tick_lo < 0x10);
1107 
1108 	end_apic_tick = apic_reg_ops->apic_read(reg);
1109 
1110 	*pit_ticks_adj = target_pit_tick - pit_tick;
1111 
1112 	intr_restore(iflag);
1113 
1114 	return (start_apic_tick - end_apic_tick);
1115 }
1116 
1117 /*
1118  * Initialise the APIC timer on the local APIC of CPU 0 to the desired
1119  * frequency.  Note at this stage in the boot sequence, the boot processor
1120  * is the only active processor.
1121  * hertz value of 0 indicates a one-shot mode request.  In this case
1122  * the function returns the resolution (in nanoseconds) for the hardware
1123  * timer interrupt.  If one-shot mode capability is not available,
1124  * the return value will be 0. apic_enable_oneshot is a global switch
1125  * for disabling the functionality.
1126  * A non-zero positive value for hertz indicates a periodic mode request.
1127  * In this case the hardware will be programmed to generate clock interrupts
1128  * at hertz frequency and returns the resolution of interrupts in
1129  * nanosecond.
1130  */
1131 
1132 int
1133 apic_clkinit(int hertz)
1134 {
1135 	int		ret;
1136 
1137 	apic_int_busy_mark = (apic_int_busy_mark *
1138 	    apic_sample_factor_redistribution) / 100;
1139 	apic_int_free_mark = (apic_int_free_mark *
1140 	    apic_sample_factor_redistribution) / 100;
1141 	apic_diff_for_redistribution = (apic_diff_for_redistribution *
1142 	    apic_sample_factor_redistribution) / 100;
1143 
1144 	ret = apic_timer_init(hertz);
1145 	return (ret);
1146 
1147 }
1148 
1149 /*
1150  * apic_preshutdown:
1151  * Called early in shutdown whilst we can still access filesystems to do
1152  * things like loading modules which will be required to complete shutdown
1153  * after filesystems are all unmounted.
1154  */
1155 void
1156 apic_preshutdown(int cmd, int fcn)
1157 {
1158 	APIC_VERBOSE_POWEROFF(("apic_preshutdown(%d,%d); m=%d a=%d\n",
1159 	    cmd, fcn, apic_poweroff_method, apic_enable_acpi));
1160 }
1161 
1162 void
1163 apic_shutdown(int cmd, int fcn)
1164 {
1165 	int restarts, attempts;
1166 	int i;
1167 	uchar_t	byte;
1168 	ulong_t iflag;
1169 
1170 	hpet_acpi_fini();
1171 
1172 	/* Send NMI to all CPUs except self to do per processor shutdown */
1173 	iflag = intr_clear();
1174 #ifdef	DEBUG
1175 	APIC_AV_PENDING_SET();
1176 #else
1177 	if (apic_mode == LOCAL_APIC)
1178 		APIC_AV_PENDING_SET();
1179 #endif /* DEBUG */
1180 	apic_shutdown_processors = 1;
1181 	apic_reg_ops->apic_write(APIC_INT_CMD1,
1182 	    AV_NMI | AV_LEVEL | AV_SH_ALL_EXCSELF);
1183 
1184 	/* restore cmos shutdown byte before reboot */
1185 	if (apic_cmos_ssb_set) {
1186 		outb(CMOS_ADDR, SSB);
1187 		outb(CMOS_DATA, 0);
1188 	}
1189 
1190 	ioapic_disable_redirection();
1191 
1192 	/*	disable apic mode if imcr present	*/
1193 	if (apic_imcrp) {
1194 		outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT);
1195 		outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_PIC);
1196 	}
1197 
1198 	apic_disable_local_apic();
1199 
1200 	intr_restore(iflag);
1201 
1202 	/* remainder of function is for shutdown cases only */
1203 	if (cmd != A_SHUTDOWN)
1204 		return;
1205 
1206 	/*
1207 	 * Switch system back into Legacy-Mode if using ACPI and
1208 	 * not powering-off.  Some BIOSes need to remain in ACPI-mode
1209 	 * for power-off to succeed (Dell Dimension 4600)
1210 	 * Do not disable ACPI while doing fastreboot
1211 	 */
1212 	if (apic_enable_acpi && fcn != AD_POWEROFF && fcn != AD_FASTREBOOT)
1213 		(void) AcpiDisable();
1214 
1215 	if (fcn == AD_FASTREBOOT) {
1216 		apic_reg_ops->apic_write(APIC_INT_CMD1,
1217 		    AV_ASSERT | AV_RESET | AV_SH_ALL_EXCSELF);
1218 	}
1219 
1220 	/* remainder of function is for shutdown+poweroff case only */
1221 	if (fcn != AD_POWEROFF)
1222 		return;
1223 
1224 	switch (apic_poweroff_method) {
1225 		case APIC_POWEROFF_VIA_RTC:
1226 
1227 			/* select the extended NVRAM bank in the RTC */
1228 			outb(CMOS_ADDR, RTC_REGA);
1229 			byte = inb(CMOS_DATA);
1230 			outb(CMOS_DATA, (byte | EXT_BANK));
1231 
1232 			outb(CMOS_ADDR, PFR_REG);
1233 
1234 			/* for Predator must toggle the PAB bit */
1235 			byte = inb(CMOS_DATA);
1236 
1237 			/*
1238 			 * clear power active bar, wakeup alarm and
1239 			 * kickstart
1240 			 */
1241 			byte &= ~(PAB_CBIT | WF_FLAG | KS_FLAG);
1242 			outb(CMOS_DATA, byte);
1243 
1244 			/* delay before next write */
1245 			drv_usecwait(1000);
1246 
1247 			/* for S40 the following would suffice */
1248 			byte = inb(CMOS_DATA);
1249 
1250 			/* power active bar control bit */
1251 			byte |= PAB_CBIT;
1252 			outb(CMOS_DATA, byte);
1253 
1254 			break;
1255 
1256 		case APIC_POWEROFF_VIA_ASPEN_BMC:
1257 			restarts = 0;
1258 restart_aspen_bmc:
1259 			if (++restarts == 3)
1260 				break;
1261 			attempts = 0;
1262 			do {
1263 				byte = inb(MISMIC_FLAG_REGISTER);
1264 				byte &= MISMIC_BUSY_MASK;
1265 				if (byte != 0) {
1266 					drv_usecwait(1000);
1267 					if (attempts >= 3)
1268 						goto restart_aspen_bmc;
1269 					++attempts;
1270 				}
1271 			} while (byte != 0);
1272 			outb(MISMIC_CNTL_REGISTER, CC_SMS_GET_STATUS);
1273 			byte = inb(MISMIC_FLAG_REGISTER);
1274 			byte |= 0x1;
1275 			outb(MISMIC_FLAG_REGISTER, byte);
1276 			i = 0;
1277 			for (; i < (sizeof (aspen_bmc)/sizeof (aspen_bmc[0]));
1278 			    i++) {
1279 				attempts = 0;
1280 				do {
1281 					byte = inb(MISMIC_FLAG_REGISTER);
1282 					byte &= MISMIC_BUSY_MASK;
1283 					if (byte != 0) {
1284 						drv_usecwait(1000);
1285 						if (attempts >= 3)
1286 							goto restart_aspen_bmc;
1287 						++attempts;
1288 					}
1289 				} while (byte != 0);
1290 				outb(MISMIC_CNTL_REGISTER, aspen_bmc[i].cntl);
1291 				outb(MISMIC_DATA_REGISTER, aspen_bmc[i].data);
1292 				byte = inb(MISMIC_FLAG_REGISTER);
1293 				byte |= 0x1;
1294 				outb(MISMIC_FLAG_REGISTER, byte);
1295 			}
1296 			break;
1297 
1298 		case APIC_POWEROFF_VIA_SITKA_BMC:
1299 			restarts = 0;
1300 restart_sitka_bmc:
1301 			if (++restarts == 3)
1302 				break;
1303 			attempts = 0;
1304 			do {
1305 				byte = inb(SMS_STATUS_REGISTER);
1306 				byte &= SMS_STATE_MASK;
1307 				if ((byte == SMS_READ_STATE) ||
1308 				    (byte == SMS_WRITE_STATE)) {
1309 					drv_usecwait(1000);
1310 					if (attempts >= 3)
1311 						goto restart_sitka_bmc;
1312 					++attempts;
1313 				}
1314 			} while ((byte == SMS_READ_STATE) ||
1315 			    (byte == SMS_WRITE_STATE));
1316 			outb(SMS_COMMAND_REGISTER, SMS_GET_STATUS);
1317 			i = 0;
1318 			for (; i < (sizeof (sitka_bmc)/sizeof (sitka_bmc[0]));
1319 			    i++) {
1320 				attempts = 0;
1321 				do {
1322 					byte = inb(SMS_STATUS_REGISTER);
1323 					byte &= SMS_IBF_MASK;
1324 					if (byte != 0) {
1325 						drv_usecwait(1000);
1326 						if (attempts >= 3)
1327 							goto restart_sitka_bmc;
1328 						++attempts;
1329 					}
1330 				} while (byte != 0);
1331 				outb(sitka_bmc[i].port, sitka_bmc[i].data);
1332 			}
1333 			break;
1334 
1335 		case APIC_POWEROFF_NONE:
1336 
1337 			/* If no APIC direct method, we will try using ACPI */
1338 			if (apic_enable_acpi) {
1339 				if (acpi_poweroff() == 1)
1340 					return;
1341 			} else
1342 				return;
1343 
1344 			break;
1345 	}
1346 	/*
1347 	 * Wait a limited time here for power to go off.
1348 	 * If the power does not go off, then there was a
1349 	 * problem and we should continue to the halt which
1350 	 * prints a message for the user to press a key to
1351 	 * reboot.
1352 	 */
1353 	drv_usecwait(7000000); /* wait seven seconds */
1354 
1355 }
1356 
1357 cyclic_id_t apic_cyclic_id;
1358 
1359 /*
1360  * The following functions are in the platform specific file so that they
1361  * can be different functions depending on whether we are running on
1362  * bare metal or a hypervisor.
1363  */
1364 
1365 /*
1366  * map an apic for memory-mapped access
1367  */
1368 uint32_t *
1369 mapin_apic(uint32_t addr, size_t len, int flags)
1370 {
1371 	return ((void *)psm_map_phys(addr, len, flags));
1372 }
1373 
1374 uint32_t *
1375 mapin_ioapic(uint32_t addr, size_t len, int flags)
1376 {
1377 	return (mapin_apic(addr, len, flags));
1378 }
1379 
1380 /*
1381  * unmap an apic
1382  */
1383 void
1384 mapout_apic(caddr_t addr, size_t len)
1385 {
1386 	psm_unmap_phys(addr, len);
1387 }
1388 
1389 void
1390 mapout_ioapic(caddr_t addr, size_t len)
1391 {
1392 	mapout_apic(addr, len);
1393 }
1394 
1395 uint32_t
1396 ioapic_read(int ioapic_ix, uint32_t reg)
1397 {
1398 	volatile uint32_t *ioapic;
1399 
1400 	ioapic = apicioadr[ioapic_ix];
1401 	ioapic[APIC_IO_REG] = reg;
1402 	return (ioapic[APIC_IO_DATA]);
1403 }
1404 
1405 void
1406 ioapic_write(int ioapic_ix, uint32_t reg, uint32_t value)
1407 {
1408 	volatile uint32_t *ioapic;
1409 
1410 	ioapic = apicioadr[ioapic_ix];
1411 	ioapic[APIC_IO_REG] = reg;
1412 	ioapic[APIC_IO_DATA] = value;
1413 }
1414 
1415 void
1416 ioapic_write_eoi(int ioapic_ix, uint32_t value)
1417 {
1418 	volatile uint32_t *ioapic;
1419 
1420 	ioapic = apicioadr[ioapic_ix];
1421 	ioapic[APIC_IO_EOI] = value;
1422 }
1423 
1424 /*
1425  * Round-robin algorithm to find the next CPU with interrupts enabled.
1426  * It can't share the same static variable apic_next_bind_cpu with
1427  * apic_get_next_bind_cpu(), since that will cause all interrupts to be
1428  * bound to CPU1 at boot time.  During boot, only CPU0 is online with
1429  * interrupts enabled when apic_get_next_bind_cpu() and apic_find_cpu()
1430  * are called.  However, the pcplusmp driver assumes that there will be
1431  * boot_ncpus CPUs configured eventually so it tries to distribute all
1432  * interrupts among CPU0 - CPU[boot_ncpus - 1].  Thus to prevent all
1433  * interrupts being targetted at CPU1, we need to use a dedicated static
1434  * variable for find_next_cpu() instead of sharing apic_next_bind_cpu.
1435  */
1436 
1437 processorid_t
1438 apic_find_cpu(int flag)
1439 {
1440 	int i;
1441 	static processorid_t acid = 0;
1442 
1443 	/* Find the first CPU with the passed-in flag set */
1444 	for (i = 0; i < apic_nproc; i++) {
1445 		if (++acid >= apic_nproc) {
1446 			acid = 0;
1447 		}
1448 		if (apic_cpu_in_range(acid) &&
1449 		    (apic_cpus[acid].aci_status & flag)) {
1450 			break;
1451 		}
1452 	}
1453 
1454 	ASSERT((apic_cpus[acid].aci_status & flag) != 0);
1455 	return (acid);
1456 }
1457 
1458 /*
1459  * Switch between safe and x2APIC IPI sending method.
1460  * CPU may power on in xapic mode or x2apic mode. If CPU needs to send IPI to
1461  * other CPUs before entering x2APIC mode, it still needs to xAPIC method.
1462  * Before sending StartIPI to target CPU, psm_send_ipi will be changed to
1463  * apic_common_send_ipi, which detects current local APIC mode and use right
1464  * method to send IPI. If some CPUs fail to start up, apic_poweron_cnt
1465  * won't return to zero, so apic_common_send_ipi will always be used.
1466  * psm_send_ipi can't be simply changed back to x2apic_send_ipi if some CPUs
1467  * failed to start up because those failed CPUs may recover itself later at
1468  * unpredictable time.
1469  */
1470 void
1471 apic_switch_ipi_callback(boolean_t enter)
1472 {
1473 	ulong_t iflag;
1474 	struct psm_ops *pops = psmops;
1475 
1476 	iflag = intr_clear();
1477 	lock_set(&apic_mode_switch_lock);
1478 	if (enter) {
1479 		ASSERT(apic_poweron_cnt >= 0);
1480 		if (apic_poweron_cnt == 0) {
1481 			pops->psm_send_ipi = apic_common_send_ipi;
1482 			send_dirintf = pops->psm_send_ipi;
1483 		}
1484 		apic_poweron_cnt++;
1485 	} else {
1486 		ASSERT(apic_poweron_cnt > 0);
1487 		apic_poweron_cnt--;
1488 		if (apic_poweron_cnt == 0) {
1489 			pops->psm_send_ipi = x2apic_send_ipi;
1490 			send_dirintf = pops->psm_send_ipi;
1491 		}
1492 	}
1493 	lock_clear(&apic_mode_switch_lock);
1494 	intr_restore(iflag);
1495 }
1496 
1497 void
1498 apic_intrmap_init(int apic_mode)
1499 {
1500 	int suppress_brdcst_eoi = 0;
1501 
1502 	if (psm_vt_ops != NULL) {
1503 		/*
1504 		 * Since X2APIC requires the use of interrupt remapping
1505 		 * (though this is not documented explicitly in the Intel
1506 		 * documentation (yet)), initialize interrupt remapping
1507 		 * support before initializing the X2APIC unit.
1508 		 */
1509 		if (((apic_intrmap_ops_t *)psm_vt_ops)->
1510 		    apic_intrmap_init(apic_mode) == DDI_SUCCESS) {
1511 
1512 			apic_vt_ops = psm_vt_ops;
1513 
1514 			/*
1515 			 * We leverage the interrupt remapping engine to
1516 			 * suppress broadcast EOI; thus we must send the
1517 			 * directed EOI with the directed-EOI handler.
1518 			 */
1519 			if (apic_directed_EOI_supported() == 0) {
1520 				suppress_brdcst_eoi = 1;
1521 			}
1522 
1523 			apic_vt_ops->apic_intrmap_enable(suppress_brdcst_eoi);
1524 
1525 			if (apic_detect_x2apic()) {
1526 				apic_enable_x2apic();
1527 			}
1528 
1529 			if (apic_directed_EOI_supported() == 0) {
1530 				apic_set_directed_EOI_handler();
1531 			}
1532 		}
1533 	}
1534 }
1535 
1536 /*ARGSUSED*/
1537 static void
1538 apic_record_ioapic_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
1539 {
1540 	irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
1541 }
1542 
1543 /*ARGSUSED*/
1544 static void
1545 apic_record_msi(void *intrmap_private, msi_regs_t *mregs)
1546 {
1547 	mregs->mr_addr = MSI_ADDR_HDR |
1548 	    (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
1549 	    (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
1550 	    (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
1551 	mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
1552 	    mregs->mr_data;
1553 }
1554 
1555 /*
1556  * Functions from apic_introp.c
1557  *
1558  * Those functions are used by apic_intr_ops().
1559  */
1560 
1561 /*
1562  * MSI support flag:
1563  * reflects whether MSI is supported at APIC level
1564  * it can also be patched through /etc/system
1565  *
1566  *  0 = default value - don't know and need to call apic_check_msi_support()
1567  *      to find out then set it accordingly
1568  *  1 = supported
1569  * -1 = not supported
1570  */
1571 int	apic_support_msi = 0;
1572 
1573 /* Multiple vector support for MSI-X */
1574 int	apic_msix_enable = 1;
1575 
1576 /* Multiple vector support for MSI */
1577 int	apic_multi_msi_enable = 1;
1578 
1579 /*
1580  * check whether the system supports MSI
1581  *
1582  * If PCI-E capability is found, then this must be a PCI-E system.
1583  * Since MSI is required for PCI-E system, it returns PSM_SUCCESS
1584  * to indicate this system supports MSI.
1585  */
1586 int
1587 apic_check_msi_support()
1588 {
1589 	dev_info_t *cdip;
1590 	char dev_type[16];
1591 	int dev_len;
1592 
1593 	DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support:\n"));
1594 
1595 	/*
1596 	 * check whether the first level children of root_node have
1597 	 * PCI-E capability
1598 	 */
1599 	for (cdip = ddi_get_child(ddi_root_node()); cdip != NULL;
1600 	    cdip = ddi_get_next_sibling(cdip)) {
1601 
1602 		DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: cdip: 0x%p,"
1603 		    " driver: %s, binding: %s, nodename: %s\n", (void *)cdip,
1604 		    ddi_driver_name(cdip), ddi_binding_name(cdip),
1605 		    ddi_node_name(cdip)));
1606 		dev_len = sizeof (dev_type);
1607 		if (ddi_getlongprop_buf(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS,
1608 		    "device_type", (caddr_t)dev_type, &dev_len)
1609 		    != DDI_PROP_SUCCESS)
1610 			continue;
1611 		if (strcmp(dev_type, "pciex") == 0)
1612 			return (PSM_SUCCESS);
1613 	}
1614 
1615 	/* MSI is not supported on this system */
1616 	DDI_INTR_IMPLDBG((CE_CONT, "apic_check_msi_support: no 'pciex' "
1617 	    "device_type found\n"));
1618 	return (PSM_FAILURE);
1619 }
1620 
1621 /*
1622  * apic_pci_msi_unconfigure:
1623  *
1624  * This and next two interfaces are copied from pci_intr_lib.c
1625  * Do ensure that these two files stay in sync.
1626  * These needed to be copied over here to avoid a deadlock situation on
1627  * certain mp systems that use MSI interrupts.
1628  *
1629  * IMPORTANT regards next three interfaces:
1630  * i) are called only for MSI/X interrupts.
1631  * ii) called with interrupts disabled, and must not block
1632  */
1633 void
1634 apic_pci_msi_unconfigure(dev_info_t *rdip, int type, int inum)
1635 {
1636 	ushort_t		msi_ctrl;
1637 	int			cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
1638 	ddi_acc_handle_t	handle = i_ddi_get_pci_config_handle(rdip);
1639 
1640 	ASSERT((handle != NULL) && (cap_ptr != 0));
1641 
1642 	if (type == DDI_INTR_TYPE_MSI) {
1643 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1644 		msi_ctrl &= (~PCI_MSI_MME_MASK);
1645 		pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
1646 		pci_config_put32(handle, cap_ptr + PCI_MSI_ADDR_OFFSET, 0);
1647 
1648 		if (msi_ctrl &  PCI_MSI_64BIT_MASK) {
1649 			pci_config_put16(handle,
1650 			    cap_ptr + PCI_MSI_64BIT_DATA, 0);
1651 			pci_config_put32(handle,
1652 			    cap_ptr + PCI_MSI_ADDR_OFFSET + 4, 0);
1653 		} else {
1654 			pci_config_put16(handle,
1655 			    cap_ptr + PCI_MSI_32BIT_DATA, 0);
1656 		}
1657 
1658 	} else if (type == DDI_INTR_TYPE_MSIX) {
1659 		uintptr_t	off;
1660 		uint32_t	mask;
1661 		ddi_intr_msix_t	*msix_p = i_ddi_get_msix(rdip);
1662 
1663 		ASSERT(msix_p != NULL);
1664 
1665 		/* Offset into "inum"th entry in the MSI-X table & mask it */
1666 		off = (uintptr_t)msix_p->msix_tbl_addr + (inum *
1667 		    PCI_MSIX_VECTOR_SIZE) + PCI_MSIX_VECTOR_CTRL_OFFSET;
1668 
1669 		mask = ddi_get32(msix_p->msix_tbl_hdl, (uint32_t *)off);
1670 
1671 		ddi_put32(msix_p->msix_tbl_hdl, (uint32_t *)off, (mask | 1));
1672 
1673 		/* Offset into the "inum"th entry in the MSI-X table */
1674 		off = (uintptr_t)msix_p->msix_tbl_addr +
1675 		    (inum * PCI_MSIX_VECTOR_SIZE);
1676 
1677 		/* Reset the "data" and "addr" bits */
1678 		ddi_put32(msix_p->msix_tbl_hdl,
1679 		    (uint32_t *)(off + PCI_MSIX_DATA_OFFSET), 0);
1680 		ddi_put64(msix_p->msix_tbl_hdl, (uint64_t *)off, 0);
1681 	}
1682 }
1683 
1684 /*
1685  * apic_pci_msi_disable_mode:
1686  */
1687 void
1688 apic_pci_msi_disable_mode(dev_info_t *rdip, int type)
1689 {
1690 	ushort_t		msi_ctrl;
1691 	int			cap_ptr = i_ddi_get_msi_msix_cap_ptr(rdip);
1692 	ddi_acc_handle_t	handle = i_ddi_get_pci_config_handle(rdip);
1693 
1694 	ASSERT((handle != NULL) && (cap_ptr != 0));
1695 
1696 	if (type == DDI_INTR_TYPE_MSI) {
1697 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSI_CTRL);
1698 		if (!(msi_ctrl & PCI_MSI_ENABLE_BIT))
1699 			return;
1700 
1701 		msi_ctrl &= ~PCI_MSI_ENABLE_BIT;	/* MSI disable */
1702 		pci_config_put16(handle, cap_ptr + PCI_MSI_CTRL, msi_ctrl);
1703 
1704 	} else if (type == DDI_INTR_TYPE_MSIX) {
1705 		msi_ctrl = pci_config_get16(handle, cap_ptr + PCI_MSIX_CTRL);
1706 		if (msi_ctrl & PCI_MSIX_ENABLE_BIT) {
1707 			msi_ctrl &= ~PCI_MSIX_ENABLE_BIT;
1708 			pci_config_put16(handle, cap_ptr + PCI_MSIX_CTRL,
1709 			    msi_ctrl);
1710 		}
1711 	}
1712 }
1713 
1714 uint32_t
1715 apic_get_localapicid(uint32_t cpuid)
1716 {
1717 	ASSERT(cpuid < apic_nproc && apic_cpus != NULL);
1718 
1719 	return (apic_cpus[cpuid].aci_local_id);
1720 }
1721 
1722 uchar_t
1723 apic_get_ioapicid(uchar_t ioapicindex)
1724 {
1725 	ASSERT(ioapicindex < MAX_IO_APIC);
1726 
1727 	return (apic_io_id[ioapicindex]);
1728 }
1729