1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 /* 27 * Copyright (c) 2010, Intel Corporation. 28 * All rights reserved. 29 */ 30 31 /* 32 * PSMI 1.1 extensions are supported only in 2.6 and later versions. 33 * PSMI 1.2 extensions are supported only in 2.7 and later versions. 34 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10. 35 * PSMI 1.5 extensions are supported in Solaris Nevada. 36 * PSMI 1.6 extensions are supported in Solaris Nevada. 37 * PSMI 1.7 extensions are supported in Solaris Nevada. 38 */ 39 #define PSMI_1_7 40 41 #include <sys/processor.h> 42 #include <sys/time.h> 43 #include <sys/psm.h> 44 #include <sys/smp_impldefs.h> 45 #include <sys/cram.h> 46 #include <sys/acpi/acpi.h> 47 #include <sys/acpica.h> 48 #include <sys/psm_common.h> 49 #include <sys/apic.h> 50 #include <sys/pit.h> 51 #include <sys/ddi.h> 52 #include <sys/sunddi.h> 53 #include <sys/ddi_impldefs.h> 54 #include <sys/pci.h> 55 #include <sys/promif.h> 56 #include <sys/x86_archext.h> 57 #include <sys/cpc_impl.h> 58 #include <sys/uadmin.h> 59 #include <sys/panic.h> 60 #include <sys/debug.h> 61 #include <sys/archsystm.h> 62 #include <sys/trap.h> 63 #include <sys/machsystm.h> 64 #include <sys/sysmacros.h> 65 #include <sys/cpuvar.h> 66 #include <sys/rm_platter.h> 67 #include <sys/privregs.h> 68 #include <sys/note.h> 69 #include <sys/pci_intr_lib.h> 70 #include <sys/spl.h> 71 #include <sys/clock.h> 72 #include <sys/dditypes.h> 73 #include <sys/sunddi.h> 74 #include <sys/x_call.h> 75 #include <sys/reboot.h> 76 #include <sys/hpet.h> 77 78 /* 79 * Local Function Prototypes 80 */ 81 static void apic_init_intr(); 82 static void apic_nmi_intr(caddr_t arg, struct regs *rp); 83 84 /* 85 * standard MP entries 86 */ 87 static int apic_probe(); 88 static int apic_clkinit(); 89 static int apic_getclkirq(int ipl); 90 static uint_t apic_calibrate(volatile uint32_t *addr, 91 uint16_t *pit_ticks_adj); 92 static hrtime_t apic_gettime(); 93 static hrtime_t apic_gethrtime(); 94 static void apic_init(); 95 static void apic_picinit(void); 96 static int apic_cpu_start(processorid_t cpuid, caddr_t ctx); 97 static int apic_cpu_stop(processorid_t cpuid, caddr_t ctx); 98 static int apic_cpu_add(psm_cpu_request_t *reqp); 99 static int apic_cpu_remove(psm_cpu_request_t *reqp); 100 static int apic_cpu_ops(psm_cpu_request_t *reqp); 101 static int apic_post_cpu_start(void); 102 static void apic_send_ipi(int cpun, int ipl); 103 static void apic_set_idlecpu(processorid_t cpun); 104 static void apic_unset_idlecpu(processorid_t cpun); 105 static int apic_intr_enter(int ipl, int *vect); 106 static void apic_setspl(int ipl); 107 static void x2apic_setspl(int ipl); 108 static void apic_switch_ipi_callback(boolean_t enter); 109 static int apic_addspl(int ipl, int vector, int min_ipl, int max_ipl); 110 static int apic_delspl(int ipl, int vector, int min_ipl, int max_ipl); 111 static void apic_shutdown(int cmd, int fcn); 112 static void apic_preshutdown(int cmd, int fcn); 113 static int apic_disable_intr(processorid_t cpun); 114 static void apic_enable_intr(processorid_t cpun); 115 static processorid_t apic_get_next_processorid(processorid_t cpun); 116 static int apic_get_ipivect(int ipl, int type); 117 static void apic_timer_reprogram(hrtime_t time); 118 static void apic_timer_enable(void); 119 static void apic_timer_disable(void); 120 static void apic_post_cyclic_setup(void *arg); 121 static void apic_intrmap_init(int apic_mode); 122 static void apic_record_ioapic_rdt(apic_irq_t *irq_ptr, ioapic_rdt_t *irdt); 123 static void apic_record_msi(apic_irq_t *irq_ptr, msi_regs_t *mregs); 124 125 static int apic_oneshot = 0; 126 int apic_oneshot_enable = 1; /* to allow disabling one-shot capability */ 127 128 /* Now the ones for Dynamic Interrupt distribution */ 129 int apic_enable_dynamic_migration = 0; 130 131 extern int apic_have_32bit_cr8; 132 133 /* 134 * These variables are frequently accessed in apic_intr_enter(), 135 * apic_intr_exit and apic_setspl, so group them together 136 */ 137 volatile uint32_t *apicadr = NULL; /* virtual addr of local APIC */ 138 int apic_setspl_delay = 1; /* apic_setspl - delay enable */ 139 int apic_clkvect; 140 141 /* vector at which error interrupts come in */ 142 int apic_errvect; 143 int apic_enable_error_intr = 1; 144 int apic_error_display_delay = 100; 145 146 /* vector at which performance counter overflow interrupts come in */ 147 int apic_cpcovf_vect; 148 int apic_enable_cpcovf_intr = 1; 149 150 /* maximum loop count when sending Start IPIs. */ 151 int apic_sipi_max_loop_count = 0x1000; 152 153 /* vector at which CMCI interrupts come in */ 154 int apic_cmci_vect; 155 extern int cmi_enable_cmci; 156 extern void cmi_cmci_trap(void); 157 158 static kmutex_t cmci_cpu_setup_lock; /* protects cmci_cpu_setup_registered */ 159 static int cmci_cpu_setup_registered; 160 161 /* number of CPUs in power-on transition state */ 162 static int apic_poweron_cnt = 0; 163 static lock_t apic_mode_switch_lock; 164 165 /* 166 * The following vector assignments influence the value of ipltopri and 167 * vectortoipl. Note that vectors 0 - 0x1f are not used. We can program 168 * idle to 0 and IPL 0 to 0xf to differentiate idle in case 169 * we care to do so in future. Note some IPLs which are rarely used 170 * will share the vector ranges and heavily used IPLs (5 and 6) have 171 * a wide range. 172 * 173 * This array is used to initialize apic_ipls[] (in apic_init()). 174 * 175 * IPL Vector range. as passed to intr_enter 176 * 0 none. 177 * 1,2,3 0x20-0x2f 0x0-0xf 178 * 4 0x30-0x3f 0x10-0x1f 179 * 5 0x40-0x5f 0x20-0x3f 180 * 6 0x60-0x7f 0x40-0x5f 181 * 7,8,9 0x80-0x8f 0x60-0x6f 182 * 10 0x90-0x9f 0x70-0x7f 183 * 11 0xa0-0xaf 0x80-0x8f 184 * ... ... 185 * 15 0xe0-0xef 0xc0-0xcf 186 * 15 0xf0-0xff 0xd0-0xdf 187 */ 188 uchar_t apic_vectortoipl[APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL] = { 189 3, 4, 5, 5, 6, 6, 9, 10, 11, 12, 13, 14, 15, 15 190 }; 191 /* 192 * The ipl of an ISR at vector X is apic_vectortoipl[X>>4] 193 * NOTE that this is vector as passed into intr_enter which is 194 * programmed vector - 0x20 (APIC_BASE_VECT) 195 */ 196 197 uchar_t apic_ipltopri[MAXIPL + 1]; /* unix ipl to apic pri */ 198 /* The taskpri to be programmed into apic to mask given ipl */ 199 200 #if defined(__amd64) 201 uchar_t apic_cr8pri[MAXIPL + 1]; /* unix ipl to cr8 pri */ 202 #endif 203 204 /* 205 * Correlation of the hardware vector to the IPL in use, initialized 206 * from apic_vectortoipl[] in apic_init(). The final IPLs may not correlate 207 * to the IPLs in apic_vectortoipl on some systems that share interrupt lines 208 * connected to errata-stricken IOAPICs 209 */ 210 uchar_t apic_ipls[APIC_AVAIL_VECTOR]; 211 212 /* 213 * Patchable global variables. 214 */ 215 int apic_forceload = 0; 216 217 int apic_coarse_hrtime = 1; /* 0 - use accurate slow gethrtime() */ 218 /* 1 - use gettime() for performance */ 219 int apic_flat_model = 0; /* 0 - clustered. 1 - flat */ 220 int apic_enable_hwsoftint = 0; /* 0 - disable, 1 - enable */ 221 int apic_enable_bind_log = 1; /* 1 - display interrupt binding log */ 222 int apic_panic_on_nmi = 0; 223 int apic_panic_on_apic_error = 0; 224 225 int apic_verbose = 0; 226 227 /* minimum number of timer ticks to program to */ 228 int apic_min_timer_ticks = 1; 229 230 /* 231 * Local static data 232 */ 233 static struct psm_ops apic_ops = { 234 apic_probe, 235 236 apic_init, 237 apic_picinit, 238 apic_intr_enter, 239 apic_intr_exit, 240 apic_setspl, 241 apic_addspl, 242 apic_delspl, 243 apic_disable_intr, 244 apic_enable_intr, 245 (int (*)(int))NULL, /* psm_softlvl_to_irq */ 246 (void (*)(int))NULL, /* psm_set_softintr */ 247 248 apic_set_idlecpu, 249 apic_unset_idlecpu, 250 251 apic_clkinit, 252 apic_getclkirq, 253 (void (*)(void))NULL, /* psm_hrtimeinit */ 254 apic_gethrtime, 255 256 apic_get_next_processorid, 257 apic_cpu_start, 258 apic_post_cpu_start, 259 apic_shutdown, 260 apic_get_ipivect, 261 apic_send_ipi, 262 263 (int (*)(dev_info_t *, int))NULL, /* psm_translate_irq */ 264 (void (*)(int, char *))NULL, /* psm_notify_error */ 265 (void (*)(int))NULL, /* psm_notify_func */ 266 apic_timer_reprogram, 267 apic_timer_enable, 268 apic_timer_disable, 269 apic_post_cyclic_setup, 270 apic_preshutdown, 271 apic_intr_ops, /* Advanced DDI Interrupt framework */ 272 apic_state, /* save, restore apic state for S3 */ 273 apic_cpu_ops, /* CPU control interface. */ 274 }; 275 276 277 static struct psm_info apic_psm_info = { 278 PSM_INFO_VER01_7, /* version */ 279 PSM_OWN_EXCLUSIVE, /* ownership */ 280 (struct psm_ops *)&apic_ops, /* operation */ 281 APIC_PCPLUSMP_NAME, /* machine name */ 282 "pcplusmp v1.4 compatible", 283 }; 284 285 static void *apic_hdlp; 286 287 #ifdef DEBUG 288 int apic_debug = 0; 289 int apic_restrict_vector = 0; 290 291 int apic_debug_msgbuf[APIC_DEBUG_MSGBUFSIZE]; 292 int apic_debug_msgbufindex = 0; 293 294 #endif /* DEBUG */ 295 296 apic_cpus_info_t *apic_cpus; 297 298 cpuset_t apic_cpumask; 299 uint_t apic_picinit_called; 300 301 /* Flag to indicate that we need to shut down all processors */ 302 static uint_t apic_shutdown_processors; 303 304 uint_t apic_nsec_per_intr = 0; 305 306 /* 307 * apic_let_idle_redistribute can have the following values: 308 * 0 - If clock decremented it from 1 to 0, clock has to call redistribute. 309 * apic_redistribute_lock prevents multiple idle cpus from redistributing 310 */ 311 int apic_num_idle_redistributions = 0; 312 static int apic_let_idle_redistribute = 0; 313 static uint_t apic_nticks = 0; 314 static uint_t apic_skipped_redistribute = 0; 315 316 /* to gather intr data and redistribute */ 317 static void apic_redistribute_compute(void); 318 319 static uint_t last_count_read = 0; 320 static lock_t apic_gethrtime_lock; 321 volatile int apic_hrtime_stamp = 0; 322 volatile hrtime_t apic_nsec_since_boot = 0; 323 static uint_t apic_hertz_count; 324 325 uint64_t apic_ticks_per_SFnsecs; /* # of ticks in SF nsecs */ 326 327 static hrtime_t apic_nsec_max; 328 329 static hrtime_t apic_last_hrtime = 0; 330 int apic_hrtime_error = 0; 331 int apic_remote_hrterr = 0; 332 int apic_num_nmis = 0; 333 int apic_apic_error = 0; 334 int apic_num_apic_errors = 0; 335 int apic_num_cksum_errors = 0; 336 337 int apic_error = 0; 338 static int apic_cmos_ssb_set = 0; 339 340 /* use to make sure only one cpu handles the nmi */ 341 static lock_t apic_nmi_lock; 342 /* use to make sure only one cpu handles the error interrupt */ 343 static lock_t apic_error_lock; 344 345 static struct { 346 uchar_t cntl; 347 uchar_t data; 348 } aspen_bmc[] = { 349 { CC_SMS_WR_START, 0x18 }, /* NetFn/LUN */ 350 { CC_SMS_WR_NEXT, 0x24 }, /* Cmd SET_WATCHDOG_TIMER */ 351 { CC_SMS_WR_NEXT, 0x84 }, /* DataByte 1: SMS/OS no log */ 352 { CC_SMS_WR_NEXT, 0x2 }, /* DataByte 2: Power Down */ 353 { CC_SMS_WR_NEXT, 0x0 }, /* DataByte 3: no pre-timeout */ 354 { CC_SMS_WR_NEXT, 0x0 }, /* DataByte 4: timer expir. */ 355 { CC_SMS_WR_NEXT, 0xa }, /* DataByte 5: init countdown */ 356 { CC_SMS_WR_END, 0x0 }, /* DataByte 6: init countdown */ 357 358 { CC_SMS_WR_START, 0x18 }, /* NetFn/LUN */ 359 { CC_SMS_WR_END, 0x22 } /* Cmd RESET_WATCHDOG_TIMER */ 360 }; 361 362 static struct { 363 int port; 364 uchar_t data; 365 } sitka_bmc[] = { 366 { SMS_COMMAND_REGISTER, SMS_WRITE_START }, 367 { SMS_DATA_REGISTER, 0x18 }, /* NetFn/LUN */ 368 { SMS_DATA_REGISTER, 0x24 }, /* Cmd SET_WATCHDOG_TIMER */ 369 { SMS_DATA_REGISTER, 0x84 }, /* DataByte 1: SMS/OS no log */ 370 { SMS_DATA_REGISTER, 0x2 }, /* DataByte 2: Power Down */ 371 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 3: no pre-timeout */ 372 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 4: timer expir. */ 373 { SMS_DATA_REGISTER, 0xa }, /* DataByte 5: init countdown */ 374 { SMS_COMMAND_REGISTER, SMS_WRITE_END }, 375 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 6: init countdown */ 376 377 { SMS_COMMAND_REGISTER, SMS_WRITE_START }, 378 { SMS_DATA_REGISTER, 0x18 }, /* NetFn/LUN */ 379 { SMS_COMMAND_REGISTER, SMS_WRITE_END }, 380 { SMS_DATA_REGISTER, 0x22 } /* Cmd RESET_WATCHDOG_TIMER */ 381 }; 382 383 /* Patchable global variables. */ 384 int apic_kmdb_on_nmi = 0; /* 0 - no, 1 - yes enter kmdb */ 385 uint32_t apic_divide_reg_init = 0; /* 0 - divide by 2 */ 386 387 /* default apic ops without interrupt remapping */ 388 static apic_intrmap_ops_t apic_nointrmap_ops = { 389 (int (*)(int))return_instr, 390 (void (*)(int))return_instr, 391 (void (*)(apic_irq_t *))return_instr, 392 (void (*)(apic_irq_t *, void *))return_instr, 393 (void (*)(apic_irq_t *))return_instr, 394 apic_record_ioapic_rdt, 395 apic_record_msi, 396 }; 397 398 apic_intrmap_ops_t *apic_vt_ops = &apic_nointrmap_ops; 399 400 /* 401 * This is the loadable module wrapper 402 */ 403 404 int 405 _init(void) 406 { 407 if (apic_coarse_hrtime) 408 apic_ops.psm_gethrtime = &apic_gettime; 409 return (psm_mod_init(&apic_hdlp, &apic_psm_info)); 410 } 411 412 int 413 _fini(void) 414 { 415 return (psm_mod_fini(&apic_hdlp, &apic_psm_info)); 416 } 417 418 int 419 _info(struct modinfo *modinfop) 420 { 421 return (psm_mod_info(&apic_hdlp, &apic_psm_info, modinfop)); 422 } 423 424 425 static int 426 apic_probe() 427 { 428 return (apic_probe_common(apic_psm_info.p_mach_idstring)); 429 } 430 431 void 432 apic_init() 433 { 434 int i; 435 int j = 1; 436 437 apic_ipltopri[0] = APIC_VECTOR_PER_IPL; /* leave 0 for idle */ 438 for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) { 439 if ((i < ((APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL) - 1)) && 440 (apic_vectortoipl[i + 1] == apic_vectortoipl[i])) 441 /* get to highest vector at the same ipl */ 442 continue; 443 for (; j <= apic_vectortoipl[i]; j++) { 444 apic_ipltopri[j] = (i << APIC_IPL_SHIFT) + 445 APIC_BASE_VECT; 446 } 447 } 448 for (; j < MAXIPL + 1; j++) 449 /* fill up any empty ipltopri slots */ 450 apic_ipltopri[j] = (i << APIC_IPL_SHIFT) + APIC_BASE_VECT; 451 apic_init_common(); 452 #if defined(__amd64) 453 /* 454 * Make cpu-specific interrupt info point to cr8pri vector 455 */ 456 for (i = 0; i <= MAXIPL; i++) 457 apic_cr8pri[i] = apic_ipltopri[i] >> APIC_IPL_SHIFT; 458 CPU->cpu_pri_data = apic_cr8pri; 459 #else 460 if (cpuid_have_cr8access(CPU)) 461 apic_have_32bit_cr8 = 1; 462 #endif /* __amd64 */ 463 } 464 465 /* 466 * handler for APIC Error interrupt. Just print a warning and continue 467 */ 468 static int 469 apic_error_intr() 470 { 471 uint_t error0, error1, error; 472 uint_t i; 473 474 /* 475 * We need to write before read as per 7.4.17 of system prog manual. 476 * We do both and or the results to be safe 477 */ 478 error0 = apic_reg_ops->apic_read(APIC_ERROR_STATUS); 479 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0); 480 error1 = apic_reg_ops->apic_read(APIC_ERROR_STATUS); 481 error = error0 | error1; 482 483 /* 484 * Clear the APIC error status (do this on all cpus that enter here) 485 * (two writes are required due to the semantics of accessing the 486 * error status register.) 487 */ 488 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0); 489 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0); 490 491 /* 492 * Prevent more than 1 CPU from handling error interrupt causing 493 * double printing (interleave of characters from multiple 494 * CPU's when using prom_printf) 495 */ 496 if (lock_try(&apic_error_lock) == 0) 497 return (error ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 498 if (error) { 499 #if DEBUG 500 if (apic_debug) 501 debug_enter("pcplusmp: APIC Error interrupt received"); 502 #endif /* DEBUG */ 503 if (apic_panic_on_apic_error) 504 cmn_err(CE_PANIC, 505 "APIC Error interrupt on CPU %d. Status = %x\n", 506 psm_get_cpu_id(), error); 507 else { 508 if ((error & ~APIC_CS_ERRORS) == 0) { 509 /* cksum error only */ 510 apic_error |= APIC_ERR_APIC_ERROR; 511 apic_apic_error |= error; 512 apic_num_apic_errors++; 513 apic_num_cksum_errors++; 514 } else { 515 /* 516 * prom_printf is the best shot we have of 517 * something which is problem free from 518 * high level/NMI type of interrupts 519 */ 520 prom_printf("APIC Error interrupt on CPU %d. " 521 "Status 0 = %x, Status 1 = %x\n", 522 psm_get_cpu_id(), error0, error1); 523 apic_error |= APIC_ERR_APIC_ERROR; 524 apic_apic_error |= error; 525 apic_num_apic_errors++; 526 for (i = 0; i < apic_error_display_delay; i++) { 527 tenmicrosec(); 528 } 529 /* 530 * provide more delay next time limited to 531 * roughly 1 clock tick time 532 */ 533 if (apic_error_display_delay < 500) 534 apic_error_display_delay *= 2; 535 } 536 } 537 lock_clear(&apic_error_lock); 538 return (DDI_INTR_CLAIMED); 539 } else { 540 lock_clear(&apic_error_lock); 541 return (DDI_INTR_UNCLAIMED); 542 } 543 /* NOTREACHED */ 544 } 545 546 /* 547 * Turn off the mask bit in the performance counter Local Vector Table entry. 548 */ 549 static void 550 apic_cpcovf_mask_clear(void) 551 { 552 apic_reg_ops->apic_write(APIC_PCINT_VECT, 553 (apic_reg_ops->apic_read(APIC_PCINT_VECT) & ~APIC_LVT_MASK)); 554 } 555 556 /*ARGSUSED*/ 557 static int 558 apic_cmci_enable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) 559 { 560 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect); 561 return (0); 562 } 563 564 /*ARGSUSED*/ 565 static int 566 apic_cmci_disable(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) 567 { 568 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect | AV_MASK); 569 return (0); 570 } 571 572 /*ARGSUSED*/ 573 static int 574 cmci_cpu_setup(cpu_setup_t what, int cpuid, void *arg) 575 { 576 cpuset_t cpu_set; 577 578 CPUSET_ONLY(cpu_set, cpuid); 579 580 switch (what) { 581 case CPU_ON: 582 xc_call(NULL, NULL, NULL, CPUSET2BV(cpu_set), 583 (xc_func_t)apic_cmci_enable); 584 break; 585 586 case CPU_OFF: 587 xc_call(NULL, NULL, NULL, CPUSET2BV(cpu_set), 588 (xc_func_t)apic_cmci_disable); 589 break; 590 591 default: 592 break; 593 } 594 595 return (0); 596 } 597 598 static void 599 apic_init_intr() 600 { 601 processorid_t cpun = psm_get_cpu_id(); 602 uint_t nlvt; 603 uint32_t svr = AV_UNIT_ENABLE | APIC_SPUR_INTR; 604 605 apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL); 606 607 if (apic_mode == LOCAL_APIC) { 608 /* 609 * We are running APIC in MMIO mode. 610 */ 611 if (apic_flat_model) { 612 apic_reg_ops->apic_write(APIC_FORMAT_REG, 613 APIC_FLAT_MODEL); 614 } else { 615 apic_reg_ops->apic_write(APIC_FORMAT_REG, 616 APIC_CLUSTER_MODEL); 617 } 618 619 apic_reg_ops->apic_write(APIC_DEST_REG, 620 AV_HIGH_ORDER >> cpun); 621 } 622 623 if (apic_directed_EOI_supported()) { 624 /* 625 * Setting the 12th bit in the Spurious Interrupt Vector 626 * Register suppresses broadcast EOIs generated by the local 627 * APIC. The suppression of broadcast EOIs happens only when 628 * interrupts are level-triggered. 629 */ 630 svr |= APIC_SVR_SUPPRESS_BROADCAST_EOI; 631 } 632 633 /* need to enable APIC before unmasking NMI */ 634 apic_reg_ops->apic_write(APIC_SPUR_INT_REG, svr); 635 636 /* 637 * Presence of an invalid vector with delivery mode AV_FIXED can 638 * cause an error interrupt, even if the entry is masked...so 639 * write a valid vector to LVT entries along with the mask bit 640 */ 641 642 /* All APICs have timer and LINT0/1 */ 643 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK|APIC_RESV_IRQ); 644 apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK|APIC_RESV_IRQ); 645 apic_reg_ops->apic_write(APIC_INT_VECT1, AV_NMI); /* enable NMI */ 646 647 /* 648 * On integrated APICs, the number of LVT entries is 649 * 'Max LVT entry' + 1; on 82489DX's (non-integrated 650 * APICs), nlvt is "3" (LINT0, LINT1, and timer) 651 */ 652 653 if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) { 654 nlvt = 3; 655 } else { 656 nlvt = ((apic_reg_ops->apic_read(APIC_VERS_REG) >> 16) & 657 0xFF) + 1; 658 } 659 660 if (nlvt >= 5) { 661 /* Enable performance counter overflow interrupt */ 662 663 if ((x86_feature & X86_MSR) != X86_MSR) 664 apic_enable_cpcovf_intr = 0; 665 if (apic_enable_cpcovf_intr) { 666 if (apic_cpcovf_vect == 0) { 667 int ipl = APIC_PCINT_IPL; 668 int irq = apic_get_ipivect(ipl, -1); 669 670 ASSERT(irq != -1); 671 apic_cpcovf_vect = 672 apic_irq_table[irq]->airq_vector; 673 ASSERT(apic_cpcovf_vect); 674 (void) add_avintr(NULL, ipl, 675 (avfunc)kcpc_hw_overflow_intr, 676 "apic pcint", irq, NULL, NULL, NULL, NULL); 677 kcpc_hw_overflow_intr_installed = 1; 678 kcpc_hw_enable_cpc_intr = 679 apic_cpcovf_mask_clear; 680 } 681 apic_reg_ops->apic_write(APIC_PCINT_VECT, 682 apic_cpcovf_vect); 683 } 684 } 685 686 if (nlvt >= 6) { 687 /* Only mask TM intr if the BIOS apparently doesn't use it */ 688 689 uint32_t lvtval; 690 691 lvtval = apic_reg_ops->apic_read(APIC_THERM_VECT); 692 if (((lvtval & AV_MASK) == AV_MASK) || 693 ((lvtval & AV_DELIV_MODE) != AV_SMI)) { 694 apic_reg_ops->apic_write(APIC_THERM_VECT, 695 AV_MASK|APIC_RESV_IRQ); 696 } 697 } 698 699 /* Enable error interrupt */ 700 701 if (nlvt >= 4 && apic_enable_error_intr) { 702 if (apic_errvect == 0) { 703 int ipl = 0xf; /* get highest priority intr */ 704 int irq = apic_get_ipivect(ipl, -1); 705 706 ASSERT(irq != -1); 707 apic_errvect = apic_irq_table[irq]->airq_vector; 708 ASSERT(apic_errvect); 709 /* 710 * Not PSMI compliant, but we are going to merge 711 * with ON anyway 712 */ 713 (void) add_avintr((void *)NULL, ipl, 714 (avfunc)apic_error_intr, "apic error intr", 715 irq, NULL, NULL, NULL, NULL); 716 } 717 apic_reg_ops->apic_write(APIC_ERR_VECT, apic_errvect); 718 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0); 719 apic_reg_ops->apic_write(APIC_ERROR_STATUS, 0); 720 } 721 722 /* Enable CMCI interrupt */ 723 if (cmi_enable_cmci) { 724 725 mutex_enter(&cmci_cpu_setup_lock); 726 if (cmci_cpu_setup_registered == 0) { 727 mutex_enter(&cpu_lock); 728 register_cpu_setup_func(cmci_cpu_setup, NULL); 729 mutex_exit(&cpu_lock); 730 cmci_cpu_setup_registered = 1; 731 } 732 mutex_exit(&cmci_cpu_setup_lock); 733 734 if (apic_cmci_vect == 0) { 735 int ipl = 0x2; 736 int irq = apic_get_ipivect(ipl, -1); 737 738 ASSERT(irq != -1); 739 apic_cmci_vect = apic_irq_table[irq]->airq_vector; 740 ASSERT(apic_cmci_vect); 741 742 (void) add_avintr(NULL, ipl, 743 (avfunc)cmi_cmci_trap, 744 "apic cmci intr", irq, NULL, NULL, NULL, NULL); 745 } 746 apic_reg_ops->apic_write(APIC_CMCI_VECT, apic_cmci_vect); 747 } 748 } 749 750 static void 751 apic_disable_local_apic() 752 { 753 apic_reg_ops->apic_write_task_reg(APIC_MASK_ALL); 754 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, AV_MASK); 755 756 /* local intr reg 0 */ 757 apic_reg_ops->apic_write(APIC_INT_VECT0, AV_MASK); 758 759 /* disable NMI */ 760 apic_reg_ops->apic_write(APIC_INT_VECT1, AV_MASK); 761 762 /* and error interrupt */ 763 apic_reg_ops->apic_write(APIC_ERR_VECT, AV_MASK); 764 765 /* and perf counter intr */ 766 apic_reg_ops->apic_write(APIC_PCINT_VECT, AV_MASK); 767 768 apic_reg_ops->apic_write(APIC_SPUR_INT_REG, APIC_SPUR_INTR); 769 } 770 771 static void 772 apic_picinit(void) 773 { 774 int i, j; 775 uint_t isr; 776 777 /* 778 * Initialize and enable interrupt remapping before apic 779 * hardware initialization 780 */ 781 apic_intrmap_init(apic_mode); 782 783 /* 784 * On UniSys Model 6520, the BIOS leaves vector 0x20 isr 785 * bit on without clearing it with EOI. Since softint 786 * uses vector 0x20 to interrupt itself, so softint will 787 * not work on this machine. In order to fix this problem 788 * a check is made to verify all the isr bits are clear. 789 * If not, EOIs are issued to clear the bits. 790 */ 791 for (i = 7; i >= 1; i--) { 792 isr = apic_reg_ops->apic_read(APIC_ISR_REG + (i * 4)); 793 if (isr != 0) 794 for (j = 0; ((j < 32) && (isr != 0)); j++) 795 if (isr & (1 << j)) { 796 apic_reg_ops->apic_write( 797 APIC_EOI_REG, 0); 798 isr &= ~(1 << j); 799 apic_error |= APIC_ERR_BOOT_EOI; 800 } 801 } 802 803 /* set a flag so we know we have run apic_picinit() */ 804 apic_picinit_called = 1; 805 LOCK_INIT_CLEAR(&apic_gethrtime_lock); 806 LOCK_INIT_CLEAR(&apic_ioapic_lock); 807 LOCK_INIT_CLEAR(&apic_error_lock); 808 LOCK_INIT_CLEAR(&apic_mode_switch_lock); 809 810 picsetup(); /* initialise the 8259 */ 811 812 /* add nmi handler - least priority nmi handler */ 813 LOCK_INIT_CLEAR(&apic_nmi_lock); 814 815 if (!psm_add_nmintr(0, (avfunc) apic_nmi_intr, 816 "pcplusmp NMI handler", (caddr_t)NULL)) 817 cmn_err(CE_WARN, "pcplusmp: Unable to add nmi handler"); 818 819 /* 820 * Check for directed-EOI capability in the local APIC. 821 */ 822 if (apic_directed_EOI_supported() == 1) { 823 apic_set_directed_EOI_handler(); 824 } 825 826 apic_init_intr(); 827 828 /* enable apic mode if imcr present */ 829 if (apic_imcrp) { 830 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT); 831 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_APIC); 832 } 833 834 ioapic_init_intr(IOAPIC_MASK); 835 } 836 837 static void 838 apic_cpu_send_SIPI(processorid_t cpun, boolean_t start) 839 { 840 int loop_count; 841 uint32_t vector; 842 uint_t apicid; 843 ulong_t iflag; 844 845 apicid = apic_cpus[cpun].aci_local_id; 846 847 /* 848 * Interrupts on current CPU will be disabled during the 849 * steps in order to avoid unwanted side effects from 850 * executing interrupt handlers on a problematic BIOS. 851 */ 852 iflag = intr_clear(); 853 854 if (start) { 855 outb(CMOS_ADDR, SSB); 856 outb(CMOS_DATA, BIOS_SHUTDOWN); 857 } 858 859 /* 860 * According to X2APIC specification in section '2.3.5.1' of 861 * Interrupt Command Register Semantics, the semantics of 862 * programming the Interrupt Command Register to dispatch an interrupt 863 * is simplified. A single MSR write to the 64-bit ICR is required 864 * for dispatching an interrupt. Specifically, with the 64-bit MSR 865 * interface to ICR, system software is not required to check the 866 * status of the delivery status bit prior to writing to the ICR 867 * to send an IPI. With the removal of the Delivery Status bit, 868 * system software no longer has a reason to read the ICR. It remains 869 * readable only to aid in debugging. 870 */ 871 #ifdef DEBUG 872 APIC_AV_PENDING_SET(); 873 #else 874 if (apic_mode == LOCAL_APIC) { 875 APIC_AV_PENDING_SET(); 876 } 877 #endif /* DEBUG */ 878 879 /* for integrated - make sure there is one INIT IPI in buffer */ 880 /* for external - it will wake up the cpu */ 881 apic_reg_ops->apic_write_int_cmd(apicid, AV_ASSERT | AV_RESET); 882 883 /* If only 1 CPU is installed, PENDING bit will not go low */ 884 for (loop_count = apic_sipi_max_loop_count; loop_count; loop_count--) { 885 if (apic_mode == LOCAL_APIC && 886 apic_reg_ops->apic_read(APIC_INT_CMD1) & AV_PENDING) 887 apic_ret(); 888 else 889 break; 890 } 891 892 apic_reg_ops->apic_write_int_cmd(apicid, AV_DEASSERT | AV_RESET); 893 drv_usecwait(20000); /* 20 milli sec */ 894 895 if (apic_cpus[cpun].aci_local_ver >= APIC_INTEGRATED_VERS) { 896 /* integrated apic */ 897 898 vector = (rm_platter_pa >> MMU_PAGESHIFT) & 899 (APIC_VECTOR_MASK | APIC_IPL_MASK); 900 901 /* to offset the INIT IPI queue up in the buffer */ 902 apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP); 903 drv_usecwait(200); /* 20 micro sec */ 904 905 /* 906 * send the second SIPI (Startup IPI) as recommended by Intel 907 * software development manual. 908 */ 909 apic_reg_ops->apic_write_int_cmd(apicid, vector | AV_STARTUP); 910 drv_usecwait(200); /* 20 micro sec */ 911 } 912 913 intr_restore(iflag); 914 } 915 916 /*ARGSUSED1*/ 917 static int 918 apic_cpu_start(processorid_t cpun, caddr_t arg) 919 { 920 ASSERT(MUTEX_HELD(&cpu_lock)); 921 922 if (!apic_cpu_in_range(cpun)) { 923 return (EINVAL); 924 } 925 926 /* 927 * Switch to apic_common_send_ipi for safety during starting other CPUs. 928 */ 929 if (apic_mode == LOCAL_X2APIC) { 930 apic_switch_ipi_callback(B_TRUE); 931 } 932 933 apic_cmos_ssb_set = 1; 934 apic_cpu_send_SIPI(cpun, B_TRUE); 935 936 return (0); 937 } 938 939 /* 940 * Put CPU into halted state with interrupts disabled. 941 */ 942 /*ARGSUSED1*/ 943 static int 944 apic_cpu_stop(processorid_t cpun, caddr_t arg) 945 { 946 int rc; 947 cpu_t *cp; 948 extern cpuset_t cpu_ready_set; 949 extern void cpu_idle_intercept_cpu(cpu_t *cp); 950 951 ASSERT(MUTEX_HELD(&cpu_lock)); 952 953 if (!apic_cpu_in_range(cpun)) { 954 return (EINVAL); 955 } 956 if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) { 957 return (ENOTSUP); 958 } 959 960 cp = cpu_get(cpun); 961 ASSERT(cp != NULL); 962 ASSERT((cp->cpu_flags & CPU_OFFLINE) != 0); 963 ASSERT((cp->cpu_flags & CPU_QUIESCED) != 0); 964 ASSERT((cp->cpu_flags & CPU_ENABLE) == 0); 965 966 /* Clear CPU_READY flag to disable cross calls. */ 967 cp->cpu_flags &= ~CPU_READY; 968 CPUSET_ATOMIC_DEL(cpu_ready_set, cpun); 969 rc = xc_flush_cpu(cp); 970 if (rc != 0) { 971 CPUSET_ATOMIC_ADD(cpu_ready_set, cpun); 972 cp->cpu_flags |= CPU_READY; 973 return (rc); 974 } 975 976 /* Intercept target CPU at a safe point before powering it off. */ 977 cpu_idle_intercept_cpu(cp); 978 979 apic_cpu_send_SIPI(cpun, B_FALSE); 980 cp->cpu_flags &= ~CPU_RUNNING; 981 982 return (0); 983 } 984 985 static int 986 apic_cpu_ops(psm_cpu_request_t *reqp) 987 { 988 if (reqp == NULL) { 989 return (EINVAL); 990 } 991 992 switch (reqp->pcr_cmd) { 993 case PSM_CPU_ADD: 994 return (apic_cpu_add(reqp)); 995 996 case PSM_CPU_REMOVE: 997 return (apic_cpu_remove(reqp)); 998 999 case PSM_CPU_STOP: 1000 return (apic_cpu_stop(reqp->req.cpu_stop.cpuid, 1001 reqp->req.cpu_stop.ctx)); 1002 1003 default: 1004 return (ENOTSUP); 1005 } 1006 } 1007 1008 #ifdef DEBUG 1009 int apic_break_on_cpu = 9; 1010 int apic_stretch_interrupts = 0; 1011 int apic_stretch_ISR = 1 << 3; /* IPL of 3 matches nothing now */ 1012 1013 void 1014 apic_break() 1015 { 1016 } 1017 #endif /* DEBUG */ 1018 1019 /* 1020 * platform_intr_enter 1021 * 1022 * Called at the beginning of the interrupt service routine to 1023 * mask all level equal to and below the interrupt priority 1024 * of the interrupting vector. An EOI should be given to 1025 * the interrupt controller to enable other HW interrupts. 1026 * 1027 * Return -1 for spurious interrupts 1028 * 1029 */ 1030 /*ARGSUSED*/ 1031 static int 1032 apic_intr_enter(int ipl, int *vectorp) 1033 { 1034 uchar_t vector; 1035 int nipl; 1036 int irq; 1037 ulong_t iflag; 1038 apic_cpus_info_t *cpu_infop; 1039 1040 /* 1041 * The real vector delivered is (*vectorp + 0x20), but our caller 1042 * subtracts 0x20 from the vector before passing it to us. 1043 * (That's why APIC_BASE_VECT is 0x20.) 1044 */ 1045 vector = (uchar_t)*vectorp; 1046 1047 /* if interrupted by the clock, increment apic_nsec_since_boot */ 1048 if (vector == apic_clkvect) { 1049 if (!apic_oneshot) { 1050 /* NOTE: this is not MT aware */ 1051 apic_hrtime_stamp++; 1052 apic_nsec_since_boot += apic_nsec_per_intr; 1053 apic_hrtime_stamp++; 1054 last_count_read = apic_hertz_count; 1055 apic_redistribute_compute(); 1056 } 1057 1058 /* We will avoid all the book keeping overhead for clock */ 1059 nipl = apic_ipls[vector]; 1060 1061 *vectorp = apic_vector_to_irq[vector + APIC_BASE_VECT]; 1062 if (apic_mode == LOCAL_APIC) { 1063 #if defined(__amd64) 1064 setcr8((ulong_t)(apic_ipltopri[nipl] >> 1065 APIC_IPL_SHIFT)); 1066 #else 1067 if (apic_have_32bit_cr8) 1068 setcr8((ulong_t)(apic_ipltopri[nipl] >> 1069 APIC_IPL_SHIFT)); 1070 else 1071 LOCAL_APIC_WRITE_REG(APIC_TASK_REG, 1072 (uint32_t)apic_ipltopri[nipl]); 1073 #endif 1074 LOCAL_APIC_WRITE_REG(APIC_EOI_REG, 0); 1075 } else { 1076 X2APIC_WRITE(APIC_TASK_REG, apic_ipltopri[nipl]); 1077 X2APIC_WRITE(APIC_EOI_REG, 0); 1078 } 1079 1080 return (nipl); 1081 } 1082 1083 cpu_infop = &apic_cpus[psm_get_cpu_id()]; 1084 1085 if (vector == (APIC_SPUR_INTR - APIC_BASE_VECT)) { 1086 cpu_infop->aci_spur_cnt++; 1087 return (APIC_INT_SPURIOUS); 1088 } 1089 1090 /* Check if the vector we got is really what we need */ 1091 if (apic_revector_pending) { 1092 /* 1093 * Disable interrupts for the duration of 1094 * the vector translation to prevent a self-race for 1095 * the apic_revector_lock. This cannot be done 1096 * in apic_xlate_vector because it is recursive and 1097 * we want the vector translation to be atomic with 1098 * respect to other (higher-priority) interrupts. 1099 */ 1100 iflag = intr_clear(); 1101 vector = apic_xlate_vector(vector + APIC_BASE_VECT) - 1102 APIC_BASE_VECT; 1103 intr_restore(iflag); 1104 } 1105 1106 nipl = apic_ipls[vector]; 1107 *vectorp = irq = apic_vector_to_irq[vector + APIC_BASE_VECT]; 1108 1109 if (apic_mode == LOCAL_APIC) { 1110 #if defined(__amd64) 1111 setcr8((ulong_t)(apic_ipltopri[nipl] >> APIC_IPL_SHIFT)); 1112 #else 1113 if (apic_have_32bit_cr8) 1114 setcr8((ulong_t)(apic_ipltopri[nipl] >> 1115 APIC_IPL_SHIFT)); 1116 else 1117 LOCAL_APIC_WRITE_REG(APIC_TASK_REG, 1118 (uint32_t)apic_ipltopri[nipl]); 1119 #endif 1120 } else { 1121 X2APIC_WRITE(APIC_TASK_REG, apic_ipltopri[nipl]); 1122 } 1123 1124 cpu_infop->aci_current[nipl] = (uchar_t)irq; 1125 cpu_infop->aci_curipl = (uchar_t)nipl; 1126 cpu_infop->aci_ISR_in_progress |= 1 << nipl; 1127 1128 /* 1129 * apic_level_intr could have been assimilated into the irq struct. 1130 * but, having it as a character array is more efficient in terms of 1131 * cache usage. So, we leave it as is. 1132 */ 1133 if (!apic_level_intr[irq]) { 1134 if (apic_mode == LOCAL_APIC) { 1135 LOCAL_APIC_WRITE_REG(APIC_EOI_REG, 0); 1136 } else { 1137 X2APIC_WRITE(APIC_EOI_REG, 0); 1138 } 1139 } 1140 1141 #ifdef DEBUG 1142 APIC_DEBUG_BUF_PUT(vector); 1143 APIC_DEBUG_BUF_PUT(irq); 1144 APIC_DEBUG_BUF_PUT(nipl); 1145 APIC_DEBUG_BUF_PUT(psm_get_cpu_id()); 1146 if ((apic_stretch_interrupts) && (apic_stretch_ISR & (1 << nipl))) 1147 drv_usecwait(apic_stretch_interrupts); 1148 1149 if (apic_break_on_cpu == psm_get_cpu_id()) 1150 apic_break(); 1151 #endif /* DEBUG */ 1152 return (nipl); 1153 } 1154 1155 /* 1156 * This macro is a common code used by MMIO local apic and X2APIC 1157 * local apic. 1158 */ 1159 #define APIC_INTR_EXIT() \ 1160 { \ 1161 cpu_infop = &apic_cpus[psm_get_cpu_id()]; \ 1162 if (apic_level_intr[irq]) \ 1163 apic_reg_ops->apic_send_eoi(irq); \ 1164 cpu_infop->aci_curipl = (uchar_t)prev_ipl; \ 1165 /* ISR above current pri could not be in progress */ \ 1166 cpu_infop->aci_ISR_in_progress &= (2 << prev_ipl) - 1; \ 1167 } 1168 1169 /* 1170 * Any changes made to this function must also change X2APIC 1171 * version of intr_exit. 1172 */ 1173 void 1174 apic_intr_exit(int prev_ipl, int irq) 1175 { 1176 apic_cpus_info_t *cpu_infop; 1177 1178 #if defined(__amd64) 1179 setcr8((ulong_t)apic_cr8pri[prev_ipl]); 1180 #else 1181 if (apic_have_32bit_cr8) 1182 setcr8((ulong_t)(apic_ipltopri[prev_ipl] >> APIC_IPL_SHIFT)); 1183 else 1184 apicadr[APIC_TASK_REG] = apic_ipltopri[prev_ipl]; 1185 #endif 1186 1187 APIC_INTR_EXIT(); 1188 } 1189 1190 /* 1191 * Same as apic_intr_exit() except it uses MSR rather than MMIO 1192 * to access local apic registers. 1193 */ 1194 void 1195 x2apic_intr_exit(int prev_ipl, int irq) 1196 { 1197 apic_cpus_info_t *cpu_infop; 1198 1199 X2APIC_WRITE(APIC_TASK_REG, apic_ipltopri[prev_ipl]); 1200 APIC_INTR_EXIT(); 1201 } 1202 1203 intr_exit_fn_t 1204 psm_intr_exit_fn(void) 1205 { 1206 if (apic_mode == LOCAL_X2APIC) 1207 return (x2apic_intr_exit); 1208 1209 return (apic_intr_exit); 1210 } 1211 1212 /* 1213 * Mask all interrupts below or equal to the given IPL. 1214 * Any changes made to this function must also change X2APIC 1215 * version of setspl. 1216 */ 1217 static void 1218 apic_setspl(int ipl) 1219 { 1220 #if defined(__amd64) 1221 setcr8((ulong_t)apic_cr8pri[ipl]); 1222 #else 1223 if (apic_have_32bit_cr8) 1224 setcr8((ulong_t)(apic_ipltopri[ipl] >> APIC_IPL_SHIFT)); 1225 else 1226 apicadr[APIC_TASK_REG] = apic_ipltopri[ipl]; 1227 #endif 1228 1229 /* interrupts at ipl above this cannot be in progress */ 1230 apic_cpus[psm_get_cpu_id()].aci_ISR_in_progress &= (2 << ipl) - 1; 1231 /* 1232 * this is a patch fix for the ALR QSMP P5 machine, so that interrupts 1233 * have enough time to come in before the priority is raised again 1234 * during the idle() loop. 1235 */ 1236 if (apic_setspl_delay) 1237 (void) apic_reg_ops->apic_get_pri(); 1238 } 1239 1240 /* 1241 * X2APIC version of setspl. 1242 * Mask all interrupts below or equal to the given IPL 1243 */ 1244 static void 1245 x2apic_setspl(int ipl) 1246 { 1247 X2APIC_WRITE(APIC_TASK_REG, apic_ipltopri[ipl]); 1248 1249 /* interrupts at ipl above this cannot be in progress */ 1250 apic_cpus[psm_get_cpu_id()].aci_ISR_in_progress &= (2 << ipl) - 1; 1251 } 1252 1253 /* 1254 * generates an interprocessor interrupt to another CPU. Any changes made to 1255 * this routine must be accompanied by similar changes to 1256 * apic_common_send_ipi(). 1257 */ 1258 static void 1259 apic_send_ipi(int cpun, int ipl) 1260 { 1261 int vector; 1262 ulong_t flag; 1263 1264 vector = apic_resv_vector[ipl]; 1265 1266 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR)); 1267 1268 flag = intr_clear(); 1269 1270 APIC_AV_PENDING_SET(); 1271 1272 apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id, 1273 vector); 1274 1275 intr_restore(flag); 1276 } 1277 1278 1279 /*ARGSUSED*/ 1280 static void 1281 apic_set_idlecpu(processorid_t cpun) 1282 { 1283 } 1284 1285 /*ARGSUSED*/ 1286 static void 1287 apic_unset_idlecpu(processorid_t cpun) 1288 { 1289 } 1290 1291 1292 void 1293 apic_ret() 1294 { 1295 } 1296 1297 /* 1298 * If apic_coarse_time == 1, then apic_gettime() is used instead of 1299 * apic_gethrtime(). This is used for performance instead of accuracy. 1300 */ 1301 1302 static hrtime_t 1303 apic_gettime() 1304 { 1305 int old_hrtime_stamp; 1306 hrtime_t temp; 1307 1308 /* 1309 * In one-shot mode, we do not keep time, so if anyone 1310 * calls psm_gettime() directly, we vector over to 1311 * gethrtime(). 1312 * one-shot mode MUST NOT be enabled if this psm is the source of 1313 * hrtime. 1314 */ 1315 1316 if (apic_oneshot) 1317 return (gethrtime()); 1318 1319 1320 gettime_again: 1321 while ((old_hrtime_stamp = apic_hrtime_stamp) & 1) 1322 apic_ret(); 1323 1324 temp = apic_nsec_since_boot; 1325 1326 if (apic_hrtime_stamp != old_hrtime_stamp) { /* got an interrupt */ 1327 goto gettime_again; 1328 } 1329 return (temp); 1330 } 1331 1332 /* 1333 * Here we return the number of nanoseconds since booting. Note every 1334 * clock interrupt increments apic_nsec_since_boot by the appropriate 1335 * amount. 1336 */ 1337 static hrtime_t 1338 apic_gethrtime() 1339 { 1340 int curr_timeval, countval, elapsed_ticks; 1341 int old_hrtime_stamp, status; 1342 hrtime_t temp; 1343 uint32_t cpun; 1344 ulong_t oflags; 1345 1346 /* 1347 * In one-shot mode, we do not keep time, so if anyone 1348 * calls psm_gethrtime() directly, we vector over to 1349 * gethrtime(). 1350 * one-shot mode MUST NOT be enabled if this psm is the source of 1351 * hrtime. 1352 */ 1353 1354 if (apic_oneshot) 1355 return (gethrtime()); 1356 1357 oflags = intr_clear(); /* prevent migration */ 1358 1359 cpun = apic_reg_ops->apic_read(APIC_LID_REG); 1360 if (apic_mode == LOCAL_APIC) 1361 cpun >>= APIC_ID_BIT_OFFSET; 1362 1363 lock_set(&apic_gethrtime_lock); 1364 1365 gethrtime_again: 1366 while ((old_hrtime_stamp = apic_hrtime_stamp) & 1) 1367 apic_ret(); 1368 1369 /* 1370 * Check to see which CPU we are on. Note the time is kept on 1371 * the local APIC of CPU 0. If on CPU 0, simply read the current 1372 * counter. If on another CPU, issue a remote read command to CPU 0. 1373 */ 1374 if (cpun == apic_cpus[0].aci_local_id) { 1375 countval = apic_reg_ops->apic_read(APIC_CURR_COUNT); 1376 } else { 1377 #ifdef DEBUG 1378 APIC_AV_PENDING_SET(); 1379 #else 1380 if (apic_mode == LOCAL_APIC) 1381 APIC_AV_PENDING_SET(); 1382 #endif /* DEBUG */ 1383 1384 apic_reg_ops->apic_write_int_cmd( 1385 apic_cpus[0].aci_local_id, APIC_CURR_ADD | AV_REMOTE); 1386 1387 while ((status = apic_reg_ops->apic_read(APIC_INT_CMD1)) 1388 & AV_READ_PENDING) { 1389 apic_ret(); 1390 } 1391 1392 if (status & AV_REMOTE_STATUS) /* 1 = valid */ 1393 countval = apic_reg_ops->apic_read(APIC_REMOTE_READ); 1394 else { /* 0 = invalid */ 1395 apic_remote_hrterr++; 1396 /* 1397 * return last hrtime right now, will need more 1398 * testing if change to retry 1399 */ 1400 temp = apic_last_hrtime; 1401 1402 lock_clear(&apic_gethrtime_lock); 1403 1404 intr_restore(oflags); 1405 1406 return (temp); 1407 } 1408 } 1409 if (countval > last_count_read) 1410 countval = 0; 1411 else 1412 last_count_read = countval; 1413 1414 elapsed_ticks = apic_hertz_count - countval; 1415 1416 curr_timeval = APIC_TICKS_TO_NSECS(elapsed_ticks); 1417 temp = apic_nsec_since_boot + curr_timeval; 1418 1419 if (apic_hrtime_stamp != old_hrtime_stamp) { /* got an interrupt */ 1420 /* we might have clobbered last_count_read. Restore it */ 1421 last_count_read = apic_hertz_count; 1422 goto gethrtime_again; 1423 } 1424 1425 if (temp < apic_last_hrtime) { 1426 /* return last hrtime if error occurs */ 1427 apic_hrtime_error++; 1428 temp = apic_last_hrtime; 1429 } 1430 else 1431 apic_last_hrtime = temp; 1432 1433 lock_clear(&apic_gethrtime_lock); 1434 intr_restore(oflags); 1435 1436 return (temp); 1437 } 1438 1439 /* apic NMI handler */ 1440 /*ARGSUSED*/ 1441 static void 1442 apic_nmi_intr(caddr_t arg, struct regs *rp) 1443 { 1444 if (apic_shutdown_processors) { 1445 apic_disable_local_apic(); 1446 return; 1447 } 1448 1449 apic_error |= APIC_ERR_NMI; 1450 1451 if (!lock_try(&apic_nmi_lock)) 1452 return; 1453 apic_num_nmis++; 1454 1455 if (apic_kmdb_on_nmi && psm_debugger()) { 1456 debug_enter("NMI received: entering kmdb\n"); 1457 } else if (apic_panic_on_nmi) { 1458 /* Keep panic from entering kmdb. */ 1459 nopanicdebug = 1; 1460 panic("NMI received\n"); 1461 } else { 1462 /* 1463 * prom_printf is the best shot we have of something which is 1464 * problem free from high level/NMI type of interrupts 1465 */ 1466 prom_printf("NMI received\n"); 1467 } 1468 1469 lock_clear(&apic_nmi_lock); 1470 } 1471 1472 /*ARGSUSED*/ 1473 static int 1474 apic_addspl(int irqno, int ipl, int min_ipl, int max_ipl) 1475 { 1476 return (apic_addspl_common(irqno, ipl, min_ipl, max_ipl)); 1477 } 1478 1479 static int 1480 apic_delspl(int irqno, int ipl, int min_ipl, int max_ipl) 1481 { 1482 return (apic_delspl_common(irqno, ipl, min_ipl, max_ipl)); 1483 } 1484 1485 static int 1486 apic_post_cpu_start() 1487 { 1488 int cpun; 1489 static int cpus_started = 1; 1490 1491 /* We know this CPU + BSP started successfully. */ 1492 cpus_started++; 1493 1494 /* 1495 * On BSP we would have enabled X2APIC, if supported by processor, 1496 * in acpi_probe(), but on AP we do it here. 1497 * 1498 * We enable X2APIC mode only if BSP is running in X2APIC & the 1499 * local APIC mode of the current CPU is MMIO (xAPIC). 1500 */ 1501 if (apic_mode == LOCAL_X2APIC && apic_detect_x2apic() && 1502 apic_local_mode() == LOCAL_APIC) { 1503 apic_enable_x2apic(); 1504 } 1505 1506 /* 1507 * Switch back to x2apic IPI sending method for performance when target 1508 * CPU has entered x2apic mode. 1509 */ 1510 if (apic_mode == LOCAL_X2APIC) { 1511 apic_switch_ipi_callback(B_FALSE); 1512 } 1513 1514 splx(ipltospl(LOCK_LEVEL)); 1515 apic_init_intr(); 1516 1517 /* 1518 * since some systems don't enable the internal cache on the non-boot 1519 * cpus, so we have to enable them here 1520 */ 1521 setcr0(getcr0() & ~(CR0_CD | CR0_NW)); 1522 1523 #ifdef DEBUG 1524 APIC_AV_PENDING_SET(); 1525 #else 1526 if (apic_mode == LOCAL_APIC) 1527 APIC_AV_PENDING_SET(); 1528 #endif /* DEBUG */ 1529 1530 /* 1531 * We may be booting, or resuming from suspend; aci_status will 1532 * be APIC_CPU_INTR_ENABLE if coming from suspend, so we add the 1533 * APIC_CPU_ONLINE flag here rather than setting aci_status completely. 1534 */ 1535 cpun = psm_get_cpu_id(); 1536 apic_cpus[cpun].aci_status |= APIC_CPU_ONLINE; 1537 1538 apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init); 1539 return (PSM_SUCCESS); 1540 } 1541 1542 processorid_t 1543 apic_get_next_processorid(processorid_t cpu_id) 1544 { 1545 1546 int i; 1547 1548 if (cpu_id == -1) 1549 return ((processorid_t)0); 1550 1551 for (i = cpu_id + 1; i < NCPU; i++) { 1552 if (apic_cpu_in_range(i)) 1553 return (i); 1554 } 1555 1556 return ((processorid_t)-1); 1557 } 1558 1559 static int 1560 apic_cpu_add(psm_cpu_request_t *reqp) 1561 { 1562 int i, rv = 0; 1563 ulong_t iflag; 1564 boolean_t first = B_TRUE; 1565 uchar_t localver; 1566 uint32_t localid, procid; 1567 processorid_t cpuid = (processorid_t)-1; 1568 mach_cpu_add_arg_t *ap; 1569 1570 ASSERT(reqp != NULL); 1571 reqp->req.cpu_add.cpuid = (processorid_t)-1; 1572 1573 /* Check whether CPU hotplug is supported. */ 1574 if (!plat_dr_support_cpu() || apic_max_nproc == -1) { 1575 return (ENOTSUP); 1576 } 1577 1578 ap = (mach_cpu_add_arg_t *)reqp->req.cpu_add.argp; 1579 switch (ap->type) { 1580 case MACH_CPU_ARG_LOCAL_APIC: 1581 localid = ap->arg.apic.apic_id; 1582 procid = ap->arg.apic.proc_id; 1583 if (localid >= 255 || procid > 255) { 1584 cmn_err(CE_WARN, 1585 "!apic: apicid(%u) or procid(%u) is invalid.", 1586 localid, procid); 1587 return (EINVAL); 1588 } 1589 break; 1590 1591 case MACH_CPU_ARG_LOCAL_X2APIC: 1592 localid = ap->arg.apic.apic_id; 1593 procid = ap->arg.apic.proc_id; 1594 if (localid >= UINT32_MAX) { 1595 cmn_err(CE_WARN, 1596 "!apic: x2apicid(%u) is invalid.", localid); 1597 return (EINVAL); 1598 } else if (localid >= 255 && apic_mode == LOCAL_APIC) { 1599 cmn_err(CE_WARN, "!apic: system is in APIC mode, " 1600 "can't support x2APIC processor."); 1601 return (ENOTSUP); 1602 } 1603 break; 1604 1605 default: 1606 cmn_err(CE_WARN, 1607 "!apic: unknown argument type %d to apic_cpu_add().", 1608 ap->type); 1609 return (EINVAL); 1610 } 1611 1612 /* Use apic_ioapic_lock to sync with apic_find_next_cpu_intr. */ 1613 iflag = intr_clear(); 1614 lock_set(&apic_ioapic_lock); 1615 1616 /* Check whether local APIC id already exists. */ 1617 for (i = 0; i < apic_nproc; i++) { 1618 if (!CPU_IN_SET(apic_cpumask, i)) 1619 continue; 1620 if (apic_cpus[i].aci_local_id == localid) { 1621 lock_clear(&apic_ioapic_lock); 1622 intr_restore(iflag); 1623 cmn_err(CE_WARN, 1624 "!apic: local apic id %u already exists.", 1625 localid); 1626 return (EEXIST); 1627 } else if (apic_cpus[i].aci_processor_id == procid) { 1628 lock_clear(&apic_ioapic_lock); 1629 intr_restore(iflag); 1630 cmn_err(CE_WARN, 1631 "!apic: processor id %u already exists.", 1632 (int)procid); 1633 return (EEXIST); 1634 } 1635 1636 /* 1637 * There's no local APIC version number available in MADT table, 1638 * so assume that all CPUs are homogeneous and use local APIC 1639 * version number of the first existing CPU. 1640 */ 1641 if (first) { 1642 first = B_FALSE; 1643 localver = apic_cpus[i].aci_local_ver; 1644 } 1645 } 1646 ASSERT(first == B_FALSE); 1647 1648 /* 1649 * Try to assign the same cpuid if APIC id exists in the dirty cache. 1650 */ 1651 for (i = 0; i < apic_max_nproc; i++) { 1652 if (CPU_IN_SET(apic_cpumask, i)) { 1653 ASSERT((apic_cpus[i].aci_status & APIC_CPU_FREE) == 0); 1654 continue; 1655 } 1656 ASSERT(apic_cpus[i].aci_status & APIC_CPU_FREE); 1657 if ((apic_cpus[i].aci_status & APIC_CPU_DIRTY) && 1658 apic_cpus[i].aci_local_id == localid && 1659 apic_cpus[i].aci_processor_id == procid) { 1660 cpuid = i; 1661 break; 1662 } 1663 } 1664 1665 /* Avoid the dirty cache and allocate fresh slot if possible. */ 1666 if (cpuid == (processorid_t)-1) { 1667 for (i = 0; i < apic_max_nproc; i++) { 1668 if ((apic_cpus[i].aci_status & APIC_CPU_FREE) && 1669 (apic_cpus[i].aci_status & APIC_CPU_DIRTY) == 0) { 1670 cpuid = i; 1671 break; 1672 } 1673 } 1674 } 1675 1676 /* Try to find any free slot as last resort. */ 1677 if (cpuid == (processorid_t)-1) { 1678 for (i = 0; i < apic_max_nproc; i++) { 1679 if (apic_cpus[i].aci_status & APIC_CPU_FREE) { 1680 cpuid = i; 1681 break; 1682 } 1683 } 1684 } 1685 1686 if (cpuid == (processorid_t)-1) { 1687 lock_clear(&apic_ioapic_lock); 1688 intr_restore(iflag); 1689 cmn_err(CE_NOTE, 1690 "!apic: failed to allocate cpu id for processor %u.", 1691 procid); 1692 rv = EAGAIN; 1693 } else if (ACPI_FAILURE(acpica_map_cpu(cpuid, procid))) { 1694 lock_clear(&apic_ioapic_lock); 1695 intr_restore(iflag); 1696 cmn_err(CE_NOTE, 1697 "!apic: failed to build mapping for processor %u.", 1698 procid); 1699 rv = EBUSY; 1700 } else { 1701 ASSERT(cpuid >= 0 && cpuid < NCPU); 1702 ASSERT(cpuid < apic_max_nproc && cpuid < max_ncpus); 1703 bzero(&apic_cpus[cpuid], sizeof (apic_cpus[0])); 1704 apic_cpus[cpuid].aci_processor_id = procid; 1705 apic_cpus[cpuid].aci_local_id = localid; 1706 apic_cpus[cpuid].aci_local_ver = localver; 1707 CPUSET_ATOMIC_ADD(apic_cpumask, cpuid); 1708 if (cpuid >= apic_nproc) { 1709 apic_nproc = cpuid + 1; 1710 } 1711 lock_clear(&apic_ioapic_lock); 1712 intr_restore(iflag); 1713 reqp->req.cpu_add.cpuid = cpuid; 1714 } 1715 1716 return (rv); 1717 } 1718 1719 static int 1720 apic_cpu_remove(psm_cpu_request_t *reqp) 1721 { 1722 int i; 1723 ulong_t iflag; 1724 processorid_t cpuid; 1725 1726 /* Check whether CPU hotplug is supported. */ 1727 if (!plat_dr_support_cpu() || apic_max_nproc == -1) { 1728 return (ENOTSUP); 1729 } 1730 1731 cpuid = reqp->req.cpu_remove.cpuid; 1732 1733 /* Use apic_ioapic_lock to sync with apic_find_next_cpu_intr. */ 1734 iflag = intr_clear(); 1735 lock_set(&apic_ioapic_lock); 1736 1737 if (!apic_cpu_in_range(cpuid)) { 1738 lock_clear(&apic_ioapic_lock); 1739 intr_restore(iflag); 1740 cmn_err(CE_WARN, 1741 "!apic: cpuid %d doesn't exist in apic_cpus array.", 1742 cpuid); 1743 return (ENODEV); 1744 } 1745 ASSERT((apic_cpus[cpuid].aci_status & APIC_CPU_FREE) == 0); 1746 1747 if (ACPI_FAILURE(acpica_unmap_cpu(cpuid))) { 1748 lock_clear(&apic_ioapic_lock); 1749 intr_restore(iflag); 1750 return (ENOENT); 1751 } 1752 1753 if (cpuid == apic_nproc - 1) { 1754 /* 1755 * We are removing the highest numbered cpuid so we need to 1756 * find the next highest cpuid as the new value for apic_nproc. 1757 */ 1758 for (i = apic_nproc; i > 0; i--) { 1759 if (CPU_IN_SET(apic_cpumask, i - 1)) { 1760 apic_nproc = i; 1761 break; 1762 } 1763 } 1764 /* at least one CPU left */ 1765 ASSERT(i > 0); 1766 } 1767 CPUSET_ATOMIC_DEL(apic_cpumask, cpuid); 1768 /* mark slot as free and keep it in the dirty cache */ 1769 apic_cpus[cpuid].aci_status = APIC_CPU_FREE | APIC_CPU_DIRTY; 1770 1771 lock_clear(&apic_ioapic_lock); 1772 intr_restore(iflag); 1773 1774 return (0); 1775 } 1776 1777 /* 1778 * type == -1 indicates it is an internal request. Do not change 1779 * resv_vector for these requests 1780 */ 1781 static int 1782 apic_get_ipivect(int ipl, int type) 1783 { 1784 uchar_t vector; 1785 int irq; 1786 1787 if ((irq = apic_allocate_irq(APIC_VECTOR(ipl))) != -1) { 1788 if (vector = apic_allocate_vector(ipl, irq, 1)) { 1789 apic_irq_table[irq]->airq_mps_intr_index = 1790 RESERVE_INDEX; 1791 apic_irq_table[irq]->airq_vector = vector; 1792 if (type != -1) { 1793 apic_resv_vector[ipl] = vector; 1794 } 1795 return (irq); 1796 } 1797 } 1798 apic_error |= APIC_ERR_GET_IPIVECT_FAIL; 1799 return (-1); /* shouldn't happen */ 1800 } 1801 1802 static int 1803 apic_getclkirq(int ipl) 1804 { 1805 int irq; 1806 1807 if ((irq = apic_get_ipivect(ipl, -1)) == -1) 1808 return (-1); 1809 /* 1810 * Note the vector in apic_clkvect for per clock handling. 1811 */ 1812 apic_clkvect = apic_irq_table[irq]->airq_vector - APIC_BASE_VECT; 1813 APIC_VERBOSE_IOAPIC((CE_NOTE, "get_clkirq: vector = %x\n", 1814 apic_clkvect)); 1815 return (irq); 1816 } 1817 1818 1819 /* 1820 * Return the number of APIC clock ticks elapsed for 8245 to decrement 1821 * (APIC_TIME_COUNT + pit_ticks_adj) ticks. 1822 */ 1823 static uint_t 1824 apic_calibrate(volatile uint32_t *addr, uint16_t *pit_ticks_adj) 1825 { 1826 uint8_t pit_tick_lo; 1827 uint16_t pit_tick, target_pit_tick; 1828 uint32_t start_apic_tick, end_apic_tick; 1829 ulong_t iflag; 1830 uint32_t reg; 1831 1832 reg = addr + APIC_CURR_COUNT - apicadr; 1833 1834 iflag = intr_clear(); 1835 1836 do { 1837 pit_tick_lo = inb(PITCTR0_PORT); 1838 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo; 1839 } while (pit_tick < APIC_TIME_MIN || 1840 pit_tick_lo <= APIC_LB_MIN || pit_tick_lo >= APIC_LB_MAX); 1841 1842 /* 1843 * Wait for the 8254 to decrement by 5 ticks to ensure 1844 * we didn't start in the middle of a tick. 1845 * Compare with 0x10 for the wrap around case. 1846 */ 1847 target_pit_tick = pit_tick - 5; 1848 do { 1849 pit_tick_lo = inb(PITCTR0_PORT); 1850 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo; 1851 } while (pit_tick > target_pit_tick || pit_tick_lo < 0x10); 1852 1853 start_apic_tick = apic_reg_ops->apic_read(reg); 1854 1855 /* 1856 * Wait for the 8254 to decrement by 1857 * (APIC_TIME_COUNT + pit_ticks_adj) ticks 1858 */ 1859 target_pit_tick = pit_tick - APIC_TIME_COUNT; 1860 do { 1861 pit_tick_lo = inb(PITCTR0_PORT); 1862 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo; 1863 } while (pit_tick > target_pit_tick || pit_tick_lo < 0x10); 1864 1865 end_apic_tick = apic_reg_ops->apic_read(reg); 1866 1867 *pit_ticks_adj = target_pit_tick - pit_tick; 1868 1869 intr_restore(iflag); 1870 1871 return (start_apic_tick - end_apic_tick); 1872 } 1873 1874 /* 1875 * Initialise the APIC timer on the local APIC of CPU 0 to the desired 1876 * frequency. Note at this stage in the boot sequence, the boot processor 1877 * is the only active processor. 1878 * hertz value of 0 indicates a one-shot mode request. In this case 1879 * the function returns the resolution (in nanoseconds) for the hardware 1880 * timer interrupt. If one-shot mode capability is not available, 1881 * the return value will be 0. apic_enable_oneshot is a global switch 1882 * for disabling the functionality. 1883 * A non-zero positive value for hertz indicates a periodic mode request. 1884 * In this case the hardware will be programmed to generate clock interrupts 1885 * at hertz frequency and returns the resolution of interrupts in 1886 * nanosecond. 1887 */ 1888 1889 static int 1890 apic_clkinit(int hertz) 1891 { 1892 uint_t apic_ticks = 0; 1893 uint_t pit_ticks; 1894 int ret; 1895 uint16_t pit_ticks_adj; 1896 static int firsttime = 1; 1897 1898 if (firsttime) { 1899 /* first time calibrate on CPU0 only */ 1900 1901 apic_reg_ops->apic_write(APIC_DIVIDE_REG, apic_divide_reg_init); 1902 apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL); 1903 apic_ticks = apic_calibrate(apicadr, &pit_ticks_adj); 1904 1905 /* total number of PIT ticks corresponding to apic_ticks */ 1906 pit_ticks = APIC_TIME_COUNT + pit_ticks_adj; 1907 1908 /* 1909 * Determine the number of nanoseconds per APIC clock tick 1910 * and then determine how many APIC ticks to interrupt at the 1911 * desired frequency 1912 * apic_ticks / (pitticks / PIT_HZ) = apic_ticks_per_s 1913 * (apic_ticks * PIT_HZ) / pitticks = apic_ticks_per_s 1914 * apic_ticks_per_ns = (apic_ticks * PIT_HZ) / (pitticks * 10^9) 1915 * pic_ticks_per_SFns = 1916 * (SF * apic_ticks * PIT_HZ) / (pitticks * 10^9) 1917 */ 1918 apic_ticks_per_SFnsecs = 1919 ((SF * apic_ticks * PIT_HZ) / 1920 ((uint64_t)pit_ticks * NANOSEC)); 1921 1922 /* the interval timer initial count is 32 bit max */ 1923 apic_nsec_max = APIC_TICKS_TO_NSECS(APIC_MAXVAL); 1924 firsttime = 0; 1925 } 1926 1927 if (hertz != 0) { 1928 /* periodic */ 1929 apic_nsec_per_intr = NANOSEC / hertz; 1930 apic_hertz_count = APIC_NSECS_TO_TICKS(apic_nsec_per_intr); 1931 } 1932 1933 apic_int_busy_mark = (apic_int_busy_mark * 1934 apic_sample_factor_redistribution) / 100; 1935 apic_int_free_mark = (apic_int_free_mark * 1936 apic_sample_factor_redistribution) / 100; 1937 apic_diff_for_redistribution = (apic_diff_for_redistribution * 1938 apic_sample_factor_redistribution) / 100; 1939 1940 if (hertz == 0) { 1941 /* requested one_shot */ 1942 if (!tsc_gethrtime_enable || !apic_oneshot_enable) 1943 return (0); 1944 apic_oneshot = 1; 1945 ret = (int)APIC_TICKS_TO_NSECS(1); 1946 } else { 1947 /* program the local APIC to interrupt at the given frequency */ 1948 apic_reg_ops->apic_write(APIC_INIT_COUNT, apic_hertz_count); 1949 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 1950 (apic_clkvect + APIC_BASE_VECT) | AV_TIME); 1951 apic_oneshot = 0; 1952 ret = NANOSEC / hertz; 1953 } 1954 1955 return (ret); 1956 1957 } 1958 1959 /* 1960 * apic_preshutdown: 1961 * Called early in shutdown whilst we can still access filesystems to do 1962 * things like loading modules which will be required to complete shutdown 1963 * after filesystems are all unmounted. 1964 */ 1965 static void 1966 apic_preshutdown(int cmd, int fcn) 1967 { 1968 APIC_VERBOSE_POWEROFF(("apic_preshutdown(%d,%d); m=%d a=%d\n", 1969 cmd, fcn, apic_poweroff_method, apic_enable_acpi)); 1970 1971 if ((cmd != A_SHUTDOWN) || (fcn != AD_POWEROFF)) { 1972 return; 1973 } 1974 } 1975 1976 static void 1977 apic_shutdown(int cmd, int fcn) 1978 { 1979 int restarts, attempts; 1980 int i; 1981 uchar_t byte; 1982 ulong_t iflag; 1983 1984 hpet_acpi_fini(); 1985 1986 /* Send NMI to all CPUs except self to do per processor shutdown */ 1987 iflag = intr_clear(); 1988 #ifdef DEBUG 1989 APIC_AV_PENDING_SET(); 1990 #else 1991 if (apic_mode == LOCAL_APIC) 1992 APIC_AV_PENDING_SET(); 1993 #endif /* DEBUG */ 1994 apic_shutdown_processors = 1; 1995 apic_reg_ops->apic_write(APIC_INT_CMD1, 1996 AV_NMI | AV_LEVEL | AV_SH_ALL_EXCSELF); 1997 1998 /* restore cmos shutdown byte before reboot */ 1999 if (apic_cmos_ssb_set) { 2000 outb(CMOS_ADDR, SSB); 2001 outb(CMOS_DATA, 0); 2002 } 2003 2004 ioapic_disable_redirection(); 2005 2006 /* disable apic mode if imcr present */ 2007 if (apic_imcrp) { 2008 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT); 2009 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_PIC); 2010 } 2011 2012 apic_disable_local_apic(); 2013 2014 intr_restore(iflag); 2015 2016 /* remainder of function is for shutdown cases only */ 2017 if (cmd != A_SHUTDOWN) 2018 return; 2019 2020 /* 2021 * Switch system back into Legacy-Mode if using ACPI and 2022 * not powering-off. Some BIOSes need to remain in ACPI-mode 2023 * for power-off to succeed (Dell Dimension 4600) 2024 * Do not disable ACPI while doing fastreboot 2025 */ 2026 if (apic_enable_acpi && fcn != AD_POWEROFF && fcn != AD_FASTREBOOT) 2027 (void) AcpiDisable(); 2028 2029 if (fcn == AD_FASTREBOOT) { 2030 apic_reg_ops->apic_write(APIC_INT_CMD1, 2031 AV_ASSERT | AV_RESET | AV_SH_ALL_EXCSELF); 2032 } 2033 2034 /* remainder of function is for shutdown+poweroff case only */ 2035 if (fcn != AD_POWEROFF) 2036 return; 2037 2038 switch (apic_poweroff_method) { 2039 case APIC_POWEROFF_VIA_RTC: 2040 2041 /* select the extended NVRAM bank in the RTC */ 2042 outb(CMOS_ADDR, RTC_REGA); 2043 byte = inb(CMOS_DATA); 2044 outb(CMOS_DATA, (byte | EXT_BANK)); 2045 2046 outb(CMOS_ADDR, PFR_REG); 2047 2048 /* for Predator must toggle the PAB bit */ 2049 byte = inb(CMOS_DATA); 2050 2051 /* 2052 * clear power active bar, wakeup alarm and 2053 * kickstart 2054 */ 2055 byte &= ~(PAB_CBIT | WF_FLAG | KS_FLAG); 2056 outb(CMOS_DATA, byte); 2057 2058 /* delay before next write */ 2059 drv_usecwait(1000); 2060 2061 /* for S40 the following would suffice */ 2062 byte = inb(CMOS_DATA); 2063 2064 /* power active bar control bit */ 2065 byte |= PAB_CBIT; 2066 outb(CMOS_DATA, byte); 2067 2068 break; 2069 2070 case APIC_POWEROFF_VIA_ASPEN_BMC: 2071 restarts = 0; 2072 restart_aspen_bmc: 2073 if (++restarts == 3) 2074 break; 2075 attempts = 0; 2076 do { 2077 byte = inb(MISMIC_FLAG_REGISTER); 2078 byte &= MISMIC_BUSY_MASK; 2079 if (byte != 0) { 2080 drv_usecwait(1000); 2081 if (attempts >= 3) 2082 goto restart_aspen_bmc; 2083 ++attempts; 2084 } 2085 } while (byte != 0); 2086 outb(MISMIC_CNTL_REGISTER, CC_SMS_GET_STATUS); 2087 byte = inb(MISMIC_FLAG_REGISTER); 2088 byte |= 0x1; 2089 outb(MISMIC_FLAG_REGISTER, byte); 2090 i = 0; 2091 for (; i < (sizeof (aspen_bmc)/sizeof (aspen_bmc[0])); 2092 i++) { 2093 attempts = 0; 2094 do { 2095 byte = inb(MISMIC_FLAG_REGISTER); 2096 byte &= MISMIC_BUSY_MASK; 2097 if (byte != 0) { 2098 drv_usecwait(1000); 2099 if (attempts >= 3) 2100 goto restart_aspen_bmc; 2101 ++attempts; 2102 } 2103 } while (byte != 0); 2104 outb(MISMIC_CNTL_REGISTER, aspen_bmc[i].cntl); 2105 outb(MISMIC_DATA_REGISTER, aspen_bmc[i].data); 2106 byte = inb(MISMIC_FLAG_REGISTER); 2107 byte |= 0x1; 2108 outb(MISMIC_FLAG_REGISTER, byte); 2109 } 2110 break; 2111 2112 case APIC_POWEROFF_VIA_SITKA_BMC: 2113 restarts = 0; 2114 restart_sitka_bmc: 2115 if (++restarts == 3) 2116 break; 2117 attempts = 0; 2118 do { 2119 byte = inb(SMS_STATUS_REGISTER); 2120 byte &= SMS_STATE_MASK; 2121 if ((byte == SMS_READ_STATE) || 2122 (byte == SMS_WRITE_STATE)) { 2123 drv_usecwait(1000); 2124 if (attempts >= 3) 2125 goto restart_sitka_bmc; 2126 ++attempts; 2127 } 2128 } while ((byte == SMS_READ_STATE) || 2129 (byte == SMS_WRITE_STATE)); 2130 outb(SMS_COMMAND_REGISTER, SMS_GET_STATUS); 2131 i = 0; 2132 for (; i < (sizeof (sitka_bmc)/sizeof (sitka_bmc[0])); 2133 i++) { 2134 attempts = 0; 2135 do { 2136 byte = inb(SMS_STATUS_REGISTER); 2137 byte &= SMS_IBF_MASK; 2138 if (byte != 0) { 2139 drv_usecwait(1000); 2140 if (attempts >= 3) 2141 goto restart_sitka_bmc; 2142 ++attempts; 2143 } 2144 } while (byte != 0); 2145 outb(sitka_bmc[i].port, sitka_bmc[i].data); 2146 } 2147 break; 2148 2149 case APIC_POWEROFF_NONE: 2150 2151 /* If no APIC direct method, we will try using ACPI */ 2152 if (apic_enable_acpi) { 2153 if (acpi_poweroff() == 1) 2154 return; 2155 } else 2156 return; 2157 2158 break; 2159 } 2160 /* 2161 * Wait a limited time here for power to go off. 2162 * If the power does not go off, then there was a 2163 * problem and we should continue to the halt which 2164 * prints a message for the user to press a key to 2165 * reboot. 2166 */ 2167 drv_usecwait(7000000); /* wait seven seconds */ 2168 2169 } 2170 2171 /* 2172 * Try and disable all interrupts. We just assign interrupts to other 2173 * processors based on policy. If any were bound by user request, we 2174 * let them continue and return failure. We do not bother to check 2175 * for cache affinity while rebinding. 2176 */ 2177 2178 static int 2179 apic_disable_intr(processorid_t cpun) 2180 { 2181 int bind_cpu = 0, i, hardbound = 0; 2182 apic_irq_t *irq_ptr; 2183 ulong_t iflag; 2184 2185 iflag = intr_clear(); 2186 lock_set(&apic_ioapic_lock); 2187 2188 for (i = 0; i <= APIC_MAX_VECTOR; i++) { 2189 if (apic_reprogram_info[i].done == B_FALSE) { 2190 if (apic_reprogram_info[i].bindcpu == cpun) { 2191 /* 2192 * CPU is busy -- it's the target of 2193 * a pending reprogramming attempt 2194 */ 2195 lock_clear(&apic_ioapic_lock); 2196 intr_restore(iflag); 2197 return (PSM_FAILURE); 2198 } 2199 } 2200 } 2201 2202 apic_cpus[cpun].aci_status &= ~APIC_CPU_INTR_ENABLE; 2203 2204 apic_cpus[cpun].aci_curipl = 0; 2205 2206 i = apic_min_device_irq; 2207 for (; i <= apic_max_device_irq; i++) { 2208 /* 2209 * If there are bound interrupts on this cpu, then 2210 * rebind them to other processors. 2211 */ 2212 if ((irq_ptr = apic_irq_table[i]) != NULL) { 2213 ASSERT((irq_ptr->airq_temp_cpu == IRQ_UNBOUND) || 2214 (irq_ptr->airq_temp_cpu == IRQ_UNINIT) || 2215 (apic_cpu_in_range(irq_ptr->airq_temp_cpu))); 2216 2217 if (irq_ptr->airq_temp_cpu == (cpun | IRQ_USER_BOUND)) { 2218 hardbound = 1; 2219 continue; 2220 } 2221 2222 if (irq_ptr->airq_temp_cpu == cpun) { 2223 do { 2224 bind_cpu = apic_find_next_cpu_intr(); 2225 } while (apic_rebind_all(irq_ptr, bind_cpu)); 2226 } 2227 } 2228 } 2229 2230 lock_clear(&apic_ioapic_lock); 2231 intr_restore(iflag); 2232 2233 if (hardbound) { 2234 cmn_err(CE_WARN, "Could not disable interrupts on %d" 2235 "due to user bound interrupts", cpun); 2236 return (PSM_FAILURE); 2237 } 2238 else 2239 return (PSM_SUCCESS); 2240 } 2241 2242 /* 2243 * Bind interrupts to the CPU's local APIC. 2244 * Interrupts should not be bound to a CPU's local APIC until the CPU 2245 * is ready to receive interrupts. 2246 */ 2247 static void 2248 apic_enable_intr(processorid_t cpun) 2249 { 2250 int i; 2251 apic_irq_t *irq_ptr; 2252 ulong_t iflag; 2253 2254 iflag = intr_clear(); 2255 lock_set(&apic_ioapic_lock); 2256 2257 apic_cpus[cpun].aci_status |= APIC_CPU_INTR_ENABLE; 2258 2259 i = apic_min_device_irq; 2260 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) { 2261 if ((irq_ptr = apic_irq_table[i]) != NULL) { 2262 if ((irq_ptr->airq_cpu & ~IRQ_USER_BOUND) == cpun) { 2263 (void) apic_rebind_all(irq_ptr, 2264 irq_ptr->airq_cpu); 2265 } 2266 } 2267 } 2268 2269 lock_clear(&apic_ioapic_lock); 2270 intr_restore(iflag); 2271 } 2272 2273 2274 /* 2275 * This function will reprogram the timer. 2276 * 2277 * When in oneshot mode the argument is the absolute time in future to 2278 * generate the interrupt at. 2279 * 2280 * When in periodic mode, the argument is the interval at which the 2281 * interrupts should be generated. There is no need to support the periodic 2282 * mode timer change at this time. 2283 */ 2284 static void 2285 apic_timer_reprogram(hrtime_t time) 2286 { 2287 hrtime_t now; 2288 uint_t ticks; 2289 int64_t delta; 2290 2291 /* 2292 * We should be called from high PIL context (CBE_HIGH_PIL), 2293 * so kpreempt is disabled. 2294 */ 2295 2296 if (!apic_oneshot) { 2297 /* time is the interval for periodic mode */ 2298 ticks = APIC_NSECS_TO_TICKS(time); 2299 } else { 2300 /* one shot mode */ 2301 2302 now = gethrtime(); 2303 delta = time - now; 2304 2305 if (delta <= 0) { 2306 /* 2307 * requested to generate an interrupt in the past 2308 * generate an interrupt as soon as possible 2309 */ 2310 ticks = apic_min_timer_ticks; 2311 } else if (delta > apic_nsec_max) { 2312 /* 2313 * requested to generate an interrupt at a time 2314 * further than what we are capable of. Set to max 2315 * the hardware can handle 2316 */ 2317 2318 ticks = APIC_MAXVAL; 2319 #ifdef DEBUG 2320 cmn_err(CE_CONT, "apic_timer_reprogram, request at" 2321 " %lld too far in future, current time" 2322 " %lld \n", time, now); 2323 #endif 2324 } else 2325 ticks = APIC_NSECS_TO_TICKS(delta); 2326 } 2327 2328 if (ticks < apic_min_timer_ticks) 2329 ticks = apic_min_timer_ticks; 2330 2331 apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks); 2332 } 2333 2334 /* 2335 * This function will enable timer interrupts. 2336 */ 2337 static void 2338 apic_timer_enable(void) 2339 { 2340 /* 2341 * We should be Called from high PIL context (CBE_HIGH_PIL), 2342 * so kpreempt is disabled. 2343 */ 2344 2345 if (!apic_oneshot) { 2346 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 2347 (apic_clkvect + APIC_BASE_VECT) | AV_TIME); 2348 } else { 2349 /* one shot */ 2350 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 2351 (apic_clkvect + APIC_BASE_VECT)); 2352 } 2353 } 2354 2355 /* 2356 * This function will disable timer interrupts. 2357 */ 2358 static void 2359 apic_timer_disable(void) 2360 { 2361 /* 2362 * We should be Called from high PIL context (CBE_HIGH_PIL), 2363 * so kpreempt is disabled. 2364 */ 2365 apic_reg_ops->apic_write(APIC_LOCAL_TIMER, 2366 (apic_clkvect + APIC_BASE_VECT) | AV_MASK); 2367 } 2368 2369 /* 2370 * Set timer far into the future and return timer 2371 * current Count in nanoseconds. 2372 */ 2373 hrtime_t 2374 apic_timer_stop_count(void) 2375 { 2376 hrtime_t ns_val; 2377 int enable_val, count_val; 2378 2379 /* 2380 * Should be called with interrupts disabled. 2381 */ 2382 ASSERT(!interrupts_enabled()); 2383 2384 enable_val = apic_reg_ops->apic_read(APIC_LOCAL_TIMER); 2385 if ((enable_val & AV_MASK) == AV_MASK) 2386 return ((hrtime_t)-1); /* timer is disabled */ 2387 2388 count_val = apic_reg_ops->apic_read(APIC_CURR_COUNT); 2389 ns_val = APIC_TICKS_TO_NSECS(count_val); 2390 2391 apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL); 2392 2393 return (ns_val); 2394 } 2395 2396 /* 2397 * Reprogram timer after Deep C-State. 2398 */ 2399 void 2400 apic_timer_restart(hrtime_t time) 2401 { 2402 apic_timer_reprogram(time); 2403 } 2404 2405 ddi_periodic_t apic_periodic_id; 2406 2407 /* 2408 * If this module needs a periodic handler for the interrupt distribution, it 2409 * can be added here. The argument to the periodic handler is not currently 2410 * used, but is reserved for future. 2411 */ 2412 static void 2413 apic_post_cyclic_setup(void *arg) 2414 { 2415 _NOTE(ARGUNUSED(arg)) 2416 /* cpu_lock is held */ 2417 /* set up a periodic handler for intr redistribution */ 2418 2419 /* 2420 * In peridoc mode intr redistribution processing is done in 2421 * apic_intr_enter during clk intr processing 2422 */ 2423 if (!apic_oneshot) 2424 return; 2425 /* 2426 * Register a periodical handler for the redistribution processing. 2427 * On X86, CY_LOW_LEVEL is mapped to the level 2 interrupt, so 2428 * DDI_IPL_2 should be passed to ddi_periodic_add() here. 2429 */ 2430 apic_periodic_id = ddi_periodic_add( 2431 (void (*)(void *))apic_redistribute_compute, NULL, 2432 apic_redistribute_sample_interval, DDI_IPL_2); 2433 } 2434 2435 static void 2436 apic_redistribute_compute(void) 2437 { 2438 int i, j, max_busy; 2439 2440 if (apic_enable_dynamic_migration) { 2441 if (++apic_nticks == apic_sample_factor_redistribution) { 2442 /* 2443 * Time to call apic_intr_redistribute(). 2444 * reset apic_nticks. This will cause max_busy 2445 * to be calculated below and if it is more than 2446 * apic_int_busy, we will do the whole thing 2447 */ 2448 apic_nticks = 0; 2449 } 2450 max_busy = 0; 2451 for (i = 0; i < apic_nproc; i++) { 2452 if (!apic_cpu_in_range(i)) 2453 continue; 2454 2455 /* 2456 * Check if curipl is non zero & if ISR is in 2457 * progress 2458 */ 2459 if (((j = apic_cpus[i].aci_curipl) != 0) && 2460 (apic_cpus[i].aci_ISR_in_progress & (1 << j))) { 2461 2462 int irq; 2463 apic_cpus[i].aci_busy++; 2464 irq = apic_cpus[i].aci_current[j]; 2465 apic_irq_table[irq]->airq_busy++; 2466 } 2467 2468 if (!apic_nticks && 2469 (apic_cpus[i].aci_busy > max_busy)) 2470 max_busy = apic_cpus[i].aci_busy; 2471 } 2472 if (!apic_nticks) { 2473 if (max_busy > apic_int_busy_mark) { 2474 /* 2475 * We could make the following check be 2476 * skipped > 1 in which case, we get a 2477 * redistribution at half the busy mark (due to 2478 * double interval). Need to be able to collect 2479 * more empirical data to decide if that is a 2480 * good strategy. Punt for now. 2481 */ 2482 if (apic_skipped_redistribute) { 2483 apic_cleanup_busy(); 2484 apic_skipped_redistribute = 0; 2485 } else { 2486 apic_intr_redistribute(); 2487 } 2488 } else 2489 apic_skipped_redistribute++; 2490 } 2491 } 2492 } 2493 2494 2495 /* 2496 * The following functions are in the platform specific file so that they 2497 * can be different functions depending on whether we are running on 2498 * bare metal or a hypervisor. 2499 */ 2500 2501 /* 2502 * map an apic for memory-mapped access 2503 */ 2504 uint32_t * 2505 mapin_apic(uint32_t addr, size_t len, int flags) 2506 { 2507 /*LINTED: pointer cast may result in improper alignment */ 2508 return ((uint32_t *)psm_map_phys(addr, len, flags)); 2509 } 2510 2511 uint32_t * 2512 mapin_ioapic(uint32_t addr, size_t len, int flags) 2513 { 2514 return (mapin_apic(addr, len, flags)); 2515 } 2516 2517 /* 2518 * unmap an apic 2519 */ 2520 void 2521 mapout_apic(caddr_t addr, size_t len) 2522 { 2523 psm_unmap_phys(addr, len); 2524 } 2525 2526 void 2527 mapout_ioapic(caddr_t addr, size_t len) 2528 { 2529 mapout_apic(addr, len); 2530 } 2531 2532 /* 2533 * Check to make sure there are enough irq slots 2534 */ 2535 int 2536 apic_check_free_irqs(int count) 2537 { 2538 int i, avail; 2539 2540 avail = 0; 2541 for (i = APIC_FIRST_FREE_IRQ; i < APIC_RESV_IRQ; i++) { 2542 if ((apic_irq_table[i] == NULL) || 2543 apic_irq_table[i]->airq_mps_intr_index == FREE_INDEX) { 2544 if (++avail >= count) 2545 return (PSM_SUCCESS); 2546 } 2547 } 2548 return (PSM_FAILURE); 2549 } 2550 2551 /* 2552 * This function allocates "count" MSI vector(s) for the given "dip/pri/type" 2553 */ 2554 int 2555 apic_alloc_msi_vectors(dev_info_t *dip, int inum, int count, int pri, 2556 int behavior) 2557 { 2558 int rcount, i; 2559 uchar_t start, irqno; 2560 uint32_t cpu; 2561 major_t major; 2562 apic_irq_t *irqptr; 2563 2564 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: dip=0x%p " 2565 "inum=0x%x pri=0x%x count=0x%x behavior=%d\n", 2566 (void *)dip, inum, pri, count, behavior)); 2567 2568 if (count > 1) { 2569 if (behavior == DDI_INTR_ALLOC_STRICT && 2570 apic_multi_msi_enable == 0) 2571 return (0); 2572 if (apic_multi_msi_enable == 0) 2573 count = 1; 2574 } 2575 2576 if ((rcount = apic_navail_vector(dip, pri)) > count) 2577 rcount = count; 2578 else if (rcount == 0 || (rcount < count && 2579 behavior == DDI_INTR_ALLOC_STRICT)) 2580 return (0); 2581 2582 /* if not ISP2, then round it down */ 2583 if (!ISP2(rcount)) 2584 rcount = 1 << (highbit(rcount) - 1); 2585 2586 mutex_enter(&airq_mutex); 2587 2588 for (start = 0; rcount > 0; rcount >>= 1) { 2589 if ((start = apic_find_multi_vectors(pri, rcount)) != 0 || 2590 behavior == DDI_INTR_ALLOC_STRICT) 2591 break; 2592 } 2593 2594 if (start == 0) { 2595 /* no vector available */ 2596 mutex_exit(&airq_mutex); 2597 return (0); 2598 } 2599 2600 if (apic_check_free_irqs(rcount) == PSM_FAILURE) { 2601 /* not enough free irq slots available */ 2602 mutex_exit(&airq_mutex); 2603 return (0); 2604 } 2605 2606 major = (dip != NULL) ? ddi_driver_major(dip) : 0; 2607 for (i = 0; i < rcount; i++) { 2608 if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == 2609 (uchar_t)-1) { 2610 /* 2611 * shouldn't happen because of the 2612 * apic_check_free_irqs() check earlier 2613 */ 2614 mutex_exit(&airq_mutex); 2615 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: " 2616 "apic_allocate_irq failed\n")); 2617 return (i); 2618 } 2619 apic_max_device_irq = max(irqno, apic_max_device_irq); 2620 apic_min_device_irq = min(irqno, apic_min_device_irq); 2621 irqptr = apic_irq_table[irqno]; 2622 #ifdef DEBUG 2623 if (apic_vector_to_irq[start + i] != APIC_RESV_IRQ) 2624 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: " 2625 "apic_vector_to_irq is not APIC_RESV_IRQ\n")); 2626 #endif 2627 apic_vector_to_irq[start + i] = (uchar_t)irqno; 2628 2629 irqptr->airq_vector = (uchar_t)(start + i); 2630 irqptr->airq_ioapicindex = (uchar_t)inum; /* start */ 2631 irqptr->airq_intin_no = (uchar_t)rcount; 2632 irqptr->airq_ipl = pri; 2633 irqptr->airq_vector = start + i; 2634 irqptr->airq_origirq = (uchar_t)(inum + i); 2635 irqptr->airq_share_id = 0; 2636 irqptr->airq_mps_intr_index = MSI_INDEX; 2637 irqptr->airq_dip = dip; 2638 irqptr->airq_major = major; 2639 if (i == 0) /* they all bound to the same cpu */ 2640 cpu = irqptr->airq_cpu = apic_bind_intr(dip, irqno, 2641 0xff, 0xff); 2642 else 2643 irqptr->airq_cpu = cpu; 2644 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msi_vectors: irq=0x%x " 2645 "dip=0x%p vector=0x%x origirq=0x%x pri=0x%x\n", irqno, 2646 (void *)irqptr->airq_dip, irqptr->airq_vector, 2647 irqptr->airq_origirq, pri)); 2648 } 2649 mutex_exit(&airq_mutex); 2650 return (rcount); 2651 } 2652 2653 /* 2654 * This function allocates "count" MSI-X vector(s) for the given "dip/pri/type" 2655 */ 2656 int 2657 apic_alloc_msix_vectors(dev_info_t *dip, int inum, int count, int pri, 2658 int behavior) 2659 { 2660 int rcount, i; 2661 major_t major; 2662 2663 mutex_enter(&airq_mutex); 2664 2665 if ((rcount = apic_navail_vector(dip, pri)) > count) 2666 rcount = count; 2667 else if (rcount == 0 || (rcount < count && 2668 behavior == DDI_INTR_ALLOC_STRICT)) { 2669 rcount = 0; 2670 goto out; 2671 } 2672 2673 if (apic_check_free_irqs(rcount) == PSM_FAILURE) { 2674 /* not enough free irq slots available */ 2675 rcount = 0; 2676 goto out; 2677 } 2678 2679 major = (dip != NULL) ? ddi_driver_major(dip) : 0; 2680 for (i = 0; i < rcount; i++) { 2681 uchar_t vector, irqno; 2682 apic_irq_t *irqptr; 2683 2684 if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == 2685 (uchar_t)-1) { 2686 /* 2687 * shouldn't happen because of the 2688 * apic_check_free_irqs() check earlier 2689 */ 2690 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msix_vectors: " 2691 "apic_allocate_irq failed\n")); 2692 rcount = i; 2693 goto out; 2694 } 2695 if ((vector = apic_allocate_vector(pri, irqno, 1)) == 0) { 2696 /* 2697 * shouldn't happen because of the 2698 * apic_navail_vector() call earlier 2699 */ 2700 DDI_INTR_IMPLDBG((CE_CONT, "apic_alloc_msix_vectors: " 2701 "apic_allocate_vector failed\n")); 2702 rcount = i; 2703 goto out; 2704 } 2705 apic_max_device_irq = max(irqno, apic_max_device_irq); 2706 apic_min_device_irq = min(irqno, apic_min_device_irq); 2707 irqptr = apic_irq_table[irqno]; 2708 irqptr->airq_vector = (uchar_t)vector; 2709 irqptr->airq_ipl = pri; 2710 irqptr->airq_origirq = (uchar_t)(inum + i); 2711 irqptr->airq_share_id = 0; 2712 irqptr->airq_mps_intr_index = MSIX_INDEX; 2713 irqptr->airq_dip = dip; 2714 irqptr->airq_major = major; 2715 irqptr->airq_cpu = apic_bind_intr(dip, irqno, 0xff, 0xff); 2716 } 2717 out: 2718 mutex_exit(&airq_mutex); 2719 return (rcount); 2720 } 2721 2722 /* 2723 * Allocate a free vector for irq at ipl. Takes care of merging of multiple 2724 * IPLs into a single APIC level as well as stretching some IPLs onto multiple 2725 * levels. APIC_HI_PRI_VECTS interrupts are reserved for high priority 2726 * requests and allocated only when pri is set. 2727 */ 2728 uchar_t 2729 apic_allocate_vector(int ipl, int irq, int pri) 2730 { 2731 int lowest, highest, i; 2732 2733 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK; 2734 lowest = apic_ipltopri[ipl - 1] + APIC_VECTOR_PER_IPL; 2735 2736 if (highest < lowest) /* Both ipl and ipl - 1 map to same pri */ 2737 lowest -= APIC_VECTOR_PER_IPL; 2738 2739 #ifdef DEBUG 2740 if (apic_restrict_vector) /* for testing shared interrupt logic */ 2741 highest = lowest + apic_restrict_vector + APIC_HI_PRI_VECTS; 2742 #endif /* DEBUG */ 2743 if (pri == 0) 2744 highest -= APIC_HI_PRI_VECTS; 2745 2746 for (i = lowest; i <= highest; i++) { 2747 if (APIC_CHECK_RESERVE_VECTORS(i)) 2748 continue; 2749 if (apic_vector_to_irq[i] == APIC_RESV_IRQ) { 2750 apic_vector_to_irq[i] = (uchar_t)irq; 2751 return (i); 2752 } 2753 } 2754 2755 return (0); 2756 } 2757 2758 /* Mark vector as not being used by any irq */ 2759 void 2760 apic_free_vector(uchar_t vector) 2761 { 2762 apic_vector_to_irq[vector] = APIC_RESV_IRQ; 2763 } 2764 2765 uint32_t 2766 ioapic_read(int ioapic_ix, uint32_t reg) 2767 { 2768 volatile uint32_t *ioapic; 2769 2770 ioapic = apicioadr[ioapic_ix]; 2771 ioapic[APIC_IO_REG] = reg; 2772 return (ioapic[APIC_IO_DATA]); 2773 } 2774 2775 void 2776 ioapic_write(int ioapic_ix, uint32_t reg, uint32_t value) 2777 { 2778 volatile uint32_t *ioapic; 2779 2780 ioapic = apicioadr[ioapic_ix]; 2781 ioapic[APIC_IO_REG] = reg; 2782 ioapic[APIC_IO_DATA] = value; 2783 } 2784 2785 void 2786 ioapic_write_eoi(int ioapic_ix, uint32_t value) 2787 { 2788 volatile uint32_t *ioapic; 2789 2790 ioapic = apicioadr[ioapic_ix]; 2791 ioapic[APIC_IO_EOI] = value; 2792 } 2793 2794 /* 2795 * Call rebind to do the actual programming. 2796 * Must be called with interrupts disabled and apic_ioapic_lock held 2797 * 'p' is polymorphic -- if this function is called to process a deferred 2798 * reprogramming, p is of type 'struct ioapic_reprogram_data *', from which 2799 * the irq pointer is retrieved. If not doing deferred reprogramming, 2800 * p is of the type 'apic_irq_t *'. 2801 * 2802 * apic_ioapic_lock must be held across this call, as it protects apic_rebind 2803 * and it protects apic_find_next_cpu_intr() from a race in which a CPU can be 2804 * taken offline after a cpu is selected, but before apic_rebind is called to 2805 * bind interrupts to it. 2806 */ 2807 int 2808 apic_setup_io_intr(void *p, int irq, boolean_t deferred) 2809 { 2810 apic_irq_t *irqptr; 2811 struct ioapic_reprogram_data *drep = NULL; 2812 int rv; 2813 2814 if (deferred) { 2815 drep = (struct ioapic_reprogram_data *)p; 2816 ASSERT(drep != NULL); 2817 irqptr = drep->irqp; 2818 } else 2819 irqptr = (apic_irq_t *)p; 2820 2821 ASSERT(irqptr != NULL); 2822 2823 rv = apic_rebind(irqptr, apic_irq_table[irq]->airq_cpu, drep); 2824 if (rv) { 2825 /* 2826 * CPU is not up or interrupts are disabled. Fall back to 2827 * the first available CPU 2828 */ 2829 rv = apic_rebind(irqptr, apic_find_next_cpu_intr(), drep); 2830 } 2831 2832 return (rv); 2833 } 2834 2835 2836 uchar_t 2837 apic_modify_vector(uchar_t vector, int irq) 2838 { 2839 apic_vector_to_irq[vector] = (uchar_t)irq; 2840 return (vector); 2841 } 2842 2843 char * 2844 apic_get_apic_type() 2845 { 2846 return (apic_psm_info.p_mach_idstring); 2847 } 2848 2849 /* 2850 * Switch between safe and x2APIC IPI sending method. 2851 * CPU may power on in xapic mode or x2apic mode. If CPU needs to send IPI to 2852 * other CPUs before entering x2APIC mode, it still needs to xAPIC method. 2853 * Before sending StartIPI to target CPU, psm_send_ipi will be changed to 2854 * apic_common_send_ipi, which detects current local APIC mode and use right 2855 * method to send IPI. If some CPUs fail to start up, apic_poweron_cnt 2856 * won't return to zero, so apic_common_send_ipi will always be used. 2857 * psm_send_ipi can't be simply changed back to x2apic_send_ipi if some CPUs 2858 * failed to start up because those failed CPUs may recover itself later at 2859 * unpredictable time. 2860 */ 2861 static void 2862 apic_switch_ipi_callback(boolean_t enter) 2863 { 2864 ulong_t iflag; 2865 struct psm_ops *pops = &apic_ops; 2866 2867 iflag = intr_clear(); 2868 lock_set(&apic_mode_switch_lock); 2869 if (enter) { 2870 ASSERT(apic_poweron_cnt >= 0); 2871 if (apic_poweron_cnt == 0) { 2872 pops->psm_send_ipi = apic_common_send_ipi; 2873 send_dirintf = pops->psm_send_ipi; 2874 } 2875 apic_poweron_cnt++; 2876 } else { 2877 ASSERT(apic_poweron_cnt > 0); 2878 apic_poweron_cnt--; 2879 if (apic_poweron_cnt == 0) { 2880 pops->psm_send_ipi = x2apic_send_ipi; 2881 send_dirintf = pops->psm_send_ipi; 2882 } 2883 } 2884 lock_clear(&apic_mode_switch_lock); 2885 intr_restore(iflag); 2886 } 2887 2888 void 2889 x2apic_update_psm() 2890 { 2891 struct psm_ops *pops = &apic_ops; 2892 2893 ASSERT(pops != NULL); 2894 2895 pops->psm_intr_exit = x2apic_intr_exit; 2896 pops->psm_setspl = x2apic_setspl; 2897 pops->psm_send_ipi = x2apic_send_ipi; 2898 send_dirintf = pops->psm_send_ipi; 2899 2900 apic_mode = LOCAL_X2APIC; 2901 apic_change_ops(); 2902 } 2903 2904 static void 2905 apic_intrmap_init(int apic_mode) 2906 { 2907 int suppress_brdcst_eoi = 0; 2908 2909 if (psm_vt_ops != NULL) { 2910 /* 2911 * Since X2APIC requires the use of interrupt remapping 2912 * (though this is not documented explicitly in the Intel 2913 * documentation (yet)), initialize interrupt remapping 2914 * support before initializing the X2APIC unit. 2915 */ 2916 if (((apic_intrmap_ops_t *)psm_vt_ops)-> 2917 apic_intrmap_init(apic_mode) == DDI_SUCCESS) { 2918 2919 apic_vt_ops = psm_vt_ops; 2920 2921 /* 2922 * We leverage the interrupt remapping engine to 2923 * suppress broadcast EOI; thus we must send the 2924 * directed EOI with the directed-EOI handler. 2925 */ 2926 if (apic_directed_EOI_supported() == 0) { 2927 suppress_brdcst_eoi = 1; 2928 } 2929 2930 apic_vt_ops->apic_intrmap_enable(suppress_brdcst_eoi); 2931 2932 if (apic_detect_x2apic()) { 2933 apic_enable_x2apic(); 2934 } 2935 2936 if (apic_directed_EOI_supported() == 0) { 2937 apic_set_directed_EOI_handler(); 2938 } 2939 } 2940 } 2941 } 2942 2943 /*ARGSUSED*/ 2944 static void 2945 apic_record_ioapic_rdt(apic_irq_t *irq_ptr, ioapic_rdt_t *irdt) 2946 { 2947 irdt->ir_hi <<= APIC_ID_BIT_OFFSET; 2948 } 2949 2950 /*ARGSUSED*/ 2951 static void 2952 apic_record_msi(apic_irq_t *irq_ptr, msi_regs_t *mregs) 2953 { 2954 mregs->mr_addr = MSI_ADDR_HDR | 2955 (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) | 2956 (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) | 2957 (mregs->mr_addr << MSI_ADDR_DEST_SHIFT); 2958 mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) | 2959 mregs->mr_data; 2960 } 2961