1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * PSMI 1.1 extensions are supported only in 2.6 and later versions. 31 * PSMI 1.2 extensions are supported only in 2.7 and later versions. 32 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10. 33 * PSMI 1.5 extensions are supported in Solaris Nevada. 34 */ 35 #define PSMI_1_5 36 37 #include <sys/processor.h> 38 #include <sys/time.h> 39 #include <sys/psm.h> 40 #include <sys/smp_impldefs.h> 41 #include <sys/cram.h> 42 #include <sys/acpi/acpi.h> 43 #include <sys/acpica.h> 44 #include <sys/psm_common.h> 45 #include "apic.h" 46 #include <sys/pit.h> 47 #include <sys/ddi.h> 48 #include <sys/sunddi.h> 49 #include <sys/ddi_impldefs.h> 50 #include <sys/pci.h> 51 #include <sys/promif.h> 52 #include <sys/x86_archext.h> 53 #include <sys/cpc_impl.h> 54 #include <sys/uadmin.h> 55 #include <sys/panic.h> 56 #include <sys/debug.h> 57 #include <sys/archsystm.h> 58 #include <sys/trap.h> 59 #include <sys/machsystm.h> 60 #include <sys/cpuvar.h> 61 #include <sys/rm_platter.h> 62 #include <sys/privregs.h> 63 #include <sys/cyclic.h> 64 #include <sys/note.h> 65 #include <sys/pci_intr_lib.h> 66 67 /* 68 * Local Function Prototypes 69 */ 70 static void apic_init_intr(); 71 static void apic_ret(); 72 static int apic_handle_defconf(); 73 static int apic_parse_mpct(caddr_t mpct, int bypass); 74 static struct apic_mpfps_hdr *apic_find_fps_sig(caddr_t fptr, int size); 75 static int apic_checksum(caddr_t bptr, int len); 76 static int get_apic_cmd1(); 77 static int get_apic_pri(); 78 static int apic_find_bus_type(char *bus); 79 static int apic_find_bus(int busid); 80 static int apic_find_bus_id(int bustype); 81 static struct apic_io_intr *apic_find_io_intr(int irqno); 82 int apic_allocate_irq(int irq); 83 static int apic_find_free_irq(int start, int end); 84 static uchar_t apic_allocate_vector(int ipl, int irq, int pri); 85 static void apic_modify_vector(uchar_t vector, int irq); 86 static void apic_mark_vector(uchar_t oldvector, uchar_t newvector); 87 static uchar_t apic_xlate_vector(uchar_t oldvector); 88 static void apic_xlate_vector_free_timeout_handler(void *arg); 89 static void apic_free_vector(uchar_t vector); 90 static void apic_reprogram_timeout_handler(void *arg); 91 static int apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu, 92 int new_bind_cpu, volatile int32_t *ioapic, int intin_no, int which_irq); 93 static int apic_setup_io_intr(apic_irq_t *irqptr, int irq); 94 static int apic_setup_io_intr_deferred(apic_irq_t *irqptr, int irq); 95 static void apic_record_rdt_entry(apic_irq_t *irqptr, int irq); 96 static struct apic_io_intr *apic_find_io_intr_w_busid(int irqno, int busid); 97 static int apic_find_intin(uchar_t ioapic, uchar_t intin); 98 static int apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno, 99 int child_ipin, struct apic_io_intr **intrp); 100 static int apic_setup_irq_table(dev_info_t *dip, int irqno, 101 struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *intr_flagp, 102 int type); 103 static int apic_setup_sci_irq_table(int irqno, uchar_t ipl, 104 iflag_t *intr_flagp); 105 static void apic_nmi_intr(caddr_t arg); 106 uchar_t apic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, 107 uchar_t intin); 108 static int apic_rebind(apic_irq_t *irq_ptr, int bind_cpu, int acquire_lock, 109 int when); 110 static int apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu, int safe); 111 static void apic_intr_redistribute(); 112 static void apic_cleanup_busy(); 113 static void apic_set_pwroff_method_from_mpcnfhdr(struct apic_mp_cnf_hdr *hdrp); 114 int apic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type); 115 116 /* ACPI support routines */ 117 static int acpi_probe(void); 118 static int apic_acpi_irq_configure(acpi_psm_lnk_t *acpipsmlnkp, dev_info_t *dip, 119 int *pci_irqp, iflag_t *intr_flagp); 120 121 static int apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid, 122 int ipin, int *pci_irqp, iflag_t *intr_flagp); 123 static uchar_t acpi_find_ioapic(int irq); 124 static int acpi_intr_compatible(iflag_t iflag1, iflag_t iflag2); 125 126 /* 127 * standard MP entries 128 */ 129 static int apic_probe(); 130 static int apic_clkinit(); 131 static int apic_getclkirq(int ipl); 132 static uint_t apic_calibrate(volatile uint32_t *addr, 133 uint16_t *pit_ticks_adj); 134 static hrtime_t apic_gettime(); 135 static hrtime_t apic_gethrtime(); 136 static void apic_init(); 137 static void apic_picinit(void); 138 static void apic_cpu_start(processorid_t cpun, caddr_t rm_code); 139 static int apic_post_cpu_start(void); 140 static void apic_send_ipi(int cpun, int ipl); 141 static void apic_set_softintr(int softintr); 142 static void apic_set_idlecpu(processorid_t cpun); 143 static void apic_unset_idlecpu(processorid_t cpun); 144 static int apic_softlvl_to_irq(int ipl); 145 static int apic_intr_enter(int ipl, int *vect); 146 static void apic_intr_exit(int ipl, int vect); 147 static void apic_setspl(int ipl); 148 static int apic_addspl(int ipl, int vector, int min_ipl, int max_ipl); 149 static int apic_delspl(int ipl, int vector, int min_ipl, int max_ipl); 150 static void apic_shutdown(int cmd, int fcn); 151 static void apic_preshutdown(int cmd, int fcn); 152 static int apic_disable_intr(processorid_t cpun); 153 static void apic_enable_intr(processorid_t cpun); 154 static processorid_t apic_get_next_processorid(processorid_t cpun); 155 static int apic_get_ipivect(int ipl, int type); 156 static void apic_timer_reprogram(hrtime_t time); 157 static void apic_timer_enable(void); 158 static void apic_timer_disable(void); 159 static void apic_post_cyclic_setup(void *arg); 160 extern int apic_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, 161 psm_intr_op_t, int *); 162 163 static int apic_oneshot = 0; 164 int apic_oneshot_enable = 1; /* to allow disabling one-shot capability */ 165 166 /* 167 * These variables are frequently accessed in apic_intr_enter(), 168 * apic_intr_exit and apic_setspl, so group them together 169 */ 170 volatile uint32_t *apicadr = NULL; /* virtual addr of local APIC */ 171 int apic_setspl_delay = 1; /* apic_setspl - delay enable */ 172 int apic_clkvect; 173 174 /* ACPI SCI interrupt configuration; -1 if SCI not used */ 175 int apic_sci_vect = -1; 176 iflag_t apic_sci_flags; 177 178 /* vector at which error interrupts come in */ 179 int apic_errvect; 180 int apic_enable_error_intr = 1; 181 int apic_error_display_delay = 100; 182 183 /* vector at which performance counter overflow interrupts come in */ 184 int apic_cpcovf_vect; 185 int apic_enable_cpcovf_intr = 1; 186 187 /* Max wait time (in microsecs) for flags to clear in an RDT entry. */ 188 static int apic_max_usecs_clear_pending = 1000; 189 190 /* Amt of usecs to wait before checking if RDT flags have reset. */ 191 #define APIC_USECS_PER_WAIT_INTERVAL 100 192 193 /* Maximum number of times to retry reprogramming via the timeout */ 194 #define APIC_REPROGRAM_MAX_TIMEOUTS 10 195 196 /* timeout delay for IOAPIC delayed reprogramming */ 197 #define APIC_REPROGRAM_TIMEOUT_DELAY 5 /* microseconds */ 198 199 /* Parameter to apic_rebind(): Should reprogramming be done now or later? */ 200 #define DEFERRED 1 201 #define IMMEDIATE 0 202 203 /* 204 * number of bits per byte, from <sys/param.h> 205 */ 206 #define UCHAR_MAX ((1 << NBBY) - 1) 207 208 uchar_t apic_reserved_irqlist[MAX_ISA_IRQ]; 209 210 /* 211 * The following vector assignments influence the value of ipltopri and 212 * vectortoipl. Note that vectors 0 - 0x1f are not used. We can program 213 * idle to 0 and IPL 0 to 0x10 to differentiate idle in case 214 * we care to do so in future. Note some IPLs which are rarely used 215 * will share the vector ranges and heavily used IPLs (5 and 6) have 216 * a wide range. 217 * IPL Vector range. as passed to intr_enter 218 * 0 none. 219 * 1,2,3 0x20-0x2f 0x0-0xf 220 * 4 0x30-0x3f 0x10-0x1f 221 * 5 0x40-0x5f 0x20-0x3f 222 * 6 0x60-0x7f 0x40-0x5f 223 * 7,8,9 0x80-0x8f 0x60-0x6f 224 * 10 0x90-0x9f 0x70-0x7f 225 * 11 0xa0-0xaf 0x80-0x8f 226 * ... ... 227 * 16 0xf0-0xff 0xd0-0xdf 228 */ 229 uchar_t apic_vectortoipl[APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL] = { 230 3, 4, 5, 5, 6, 6, 9, 10, 11, 12, 13, 14, 15, 16 231 }; 232 /* 233 * The ipl of an ISR at vector X is apic_vectortoipl[X<<4] 234 * NOTE that this is vector as passed into intr_enter which is 235 * programmed vector - 0x20 (APIC_BASE_VECT) 236 */ 237 238 uchar_t apic_ipltopri[MAXIPL + 1]; /* unix ipl to apic pri */ 239 /* The taskpri to be programmed into apic to mask given ipl */ 240 241 #if defined(__amd64) 242 uchar_t apic_cr8pri[MAXIPL + 1]; /* unix ipl to cr8 pri */ 243 #endif 244 245 /* 246 * Patchable global variables. 247 */ 248 int apic_forceload = 0; 249 250 #define INTR_ROUND_ROBIN_WITH_AFFINITY 0 251 #define INTR_ROUND_ROBIN 1 252 #define INTR_LOWEST_PRIORITY 2 253 254 int apic_intr_policy = INTR_ROUND_ROBIN_WITH_AFFINITY; 255 256 static int apic_next_bind_cpu = 2; /* For round robin assignment */ 257 /* start with cpu 1 */ 258 259 int apic_coarse_hrtime = 1; /* 0 - use accurate slow gethrtime() */ 260 /* 1 - use gettime() for performance */ 261 int apic_flat_model = 0; /* 0 - clustered. 1 - flat */ 262 int apic_enable_hwsoftint = 0; /* 0 - disable, 1 - enable */ 263 int apic_enable_bind_log = 1; /* 1 - display interrupt binding log */ 264 int apic_panic_on_nmi = 0; 265 int apic_panic_on_apic_error = 0; 266 267 int apic_verbose = 0; 268 269 /* Flag definitions for apic_verbose */ 270 #define APIC_VERBOSE_IOAPIC_FLAG 0x00000001 271 #define APIC_VERBOSE_IRQ_FLAG 0x00000002 272 #define APIC_VERBOSE_POWEROFF_FLAG 0x00000004 273 #define APIC_VERBOSE_POWEROFF_PAUSE_FLAG 0x00000008 274 275 276 #define APIC_VERBOSE_IOAPIC(fmt) \ 277 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) \ 278 cmn_err fmt; 279 280 #define APIC_VERBOSE_IRQ(fmt) \ 281 if (apic_verbose & APIC_VERBOSE_IRQ_FLAG) \ 282 cmn_err fmt; 283 284 #define APIC_VERBOSE_POWEROFF(fmt) \ 285 if (apic_verbose & APIC_VERBOSE_POWEROFF_FLAG) \ 286 prom_printf fmt; 287 288 289 /* Now the ones for Dynamic Interrupt distribution */ 290 int apic_enable_dynamic_migration = 1; 291 292 /* 293 * If enabled, the distribution works as follows: 294 * On every interrupt entry, the current ipl for the CPU is set in cpu_info 295 * and the irq corresponding to the ipl is also set in the aci_current array. 296 * interrupt exit and setspl (due to soft interrupts) will cause the current 297 * ipl to be be changed. This is cache friendly as these frequently used 298 * paths write into a per cpu structure. 299 * 300 * Sampling is done by checking the structures for all CPUs and incrementing 301 * the busy field of the irq (if any) executing on each CPU and the busy field 302 * of the corresponding CPU. 303 * In periodic mode this is done on every clock interrupt. 304 * In one-shot mode, this is done thru a cyclic with an interval of 305 * apic_redistribute_sample_interval (default 10 milli sec). 306 * 307 * Every apic_sample_factor_redistribution times we sample, we do computations 308 * to decide which interrupt needs to be migrated (see comments 309 * before apic_intr_redistribute(). 310 */ 311 312 /* 313 * Following 3 variables start as % and can be patched or set using an 314 * API to be defined in future. They will be scaled to 315 * sample_factor_redistribution which is in turn set to hertz+1 (in periodic 316 * mode), or 101 in one-shot mode to stagger it away from one sec processing 317 */ 318 319 int apic_int_busy_mark = 60; 320 int apic_int_free_mark = 20; 321 int apic_diff_for_redistribution = 10; 322 323 /* sampling interval for interrupt redistribution for dynamic migration */ 324 int apic_redistribute_sample_interval = NANOSEC / 100; /* 10 millisec */ 325 326 /* 327 * number of times we sample before deciding to redistribute interrupts 328 * for dynamic migration 329 */ 330 int apic_sample_factor_redistribution = 101; 331 332 /* timeout for xlate_vector, mark_vector */ 333 int apic_revector_timeout = 16 * 10000; /* 160 millisec */ 334 335 int apic_redist_cpu_skip = 0; 336 int apic_num_imbalance = 0; 337 int apic_num_rebind = 0; 338 339 int apic_nproc = 0; 340 int apic_defconf = 0; 341 int apic_irq_translate = 0; 342 int apic_spec_rev = 0; 343 int apic_imcrp = 0; 344 345 int apic_use_acpi = 1; /* 1 = use ACPI, 0 = don't use ACPI */ 346 int apic_use_acpi_madt_only = 0; /* 1=ONLY use MADT from ACPI */ 347 348 /* 349 * For interrupt link devices, if apic_unconditional_srs is set, an irq resource 350 * will be assigned (via _SRS). If it is not set, use the current 351 * irq setting (via _CRS), but only if that irq is in the set of possible 352 * irqs (returned by _PRS) for the device. 353 */ 354 int apic_unconditional_srs = 1; 355 356 /* 357 * For interrupt link devices, if apic_prefer_crs is set when we are 358 * assigning an IRQ resource to a device, prefer the current IRQ setting 359 * over other possible irq settings under same conditions. 360 */ 361 362 int apic_prefer_crs = 1; 363 364 365 /* minimum number of timer ticks to program to */ 366 int apic_min_timer_ticks = 1; 367 /* 368 * Local static data 369 */ 370 static struct psm_ops apic_ops = { 371 apic_probe, 372 373 apic_init, 374 apic_picinit, 375 apic_intr_enter, 376 apic_intr_exit, 377 apic_setspl, 378 apic_addspl, 379 apic_delspl, 380 apic_disable_intr, 381 apic_enable_intr, 382 apic_softlvl_to_irq, 383 apic_set_softintr, 384 385 apic_set_idlecpu, 386 apic_unset_idlecpu, 387 388 apic_clkinit, 389 apic_getclkirq, 390 (void (*)(void))NULL, /* psm_hrtimeinit */ 391 apic_gethrtime, 392 393 apic_get_next_processorid, 394 apic_cpu_start, 395 apic_post_cpu_start, 396 apic_shutdown, 397 apic_get_ipivect, 398 apic_send_ipi, 399 400 (int (*)(dev_info_t *, int))NULL, /* psm_translate_irq */ 401 (int (*)(todinfo_t *))NULL, /* psm_tod_get */ 402 (int (*)(todinfo_t *))NULL, /* psm_tod_set */ 403 (void (*)(int, char *))NULL, /* psm_notify_error */ 404 (void (*)(int))NULL, /* psm_notify_func */ 405 apic_timer_reprogram, 406 apic_timer_enable, 407 apic_timer_disable, 408 apic_post_cyclic_setup, 409 apic_preshutdown, 410 apic_intr_ops /* Advanced DDI Interrupt framework */ 411 }; 412 413 414 static struct psm_info apic_psm_info = { 415 PSM_INFO_VER01_5, /* version */ 416 PSM_OWN_EXCLUSIVE, /* ownership */ 417 (struct psm_ops *)&apic_ops, /* operation */ 418 "pcplusmp", /* machine name */ 419 "pcplusmp v1.4 compatible %I%", 420 }; 421 422 static void *apic_hdlp; 423 424 #ifdef DEBUG 425 #define DENT 0x0001 426 int apic_debug = 0; 427 /* 428 * set apic_restrict_vector to the # of vectors we want to allow per range 429 * useful in testing shared interrupt logic by setting it to 2 or 3 430 */ 431 int apic_restrict_vector = 0; 432 433 #define APIC_DEBUG_MSGBUFSIZE 2048 434 int apic_debug_msgbuf[APIC_DEBUG_MSGBUFSIZE]; 435 int apic_debug_msgbufindex = 0; 436 437 /* 438 * Put "int" info into debug buffer. No MP consistency, but light weight. 439 * Good enough for most debugging. 440 */ 441 #define APIC_DEBUG_BUF_PUT(x) \ 442 apic_debug_msgbuf[apic_debug_msgbufindex++] = x; \ 443 if (apic_debug_msgbufindex >= (APIC_DEBUG_MSGBUFSIZE - NCPU)) \ 444 apic_debug_msgbufindex = 0; 445 446 #endif /* DEBUG */ 447 448 apic_cpus_info_t *apic_cpus; 449 450 static uint_t apic_cpumask = 0; 451 static uint_t apic_flag; 452 453 /* Flag to indicate that we need to shut down all processors */ 454 static uint_t apic_shutdown_processors; 455 456 uint_t apic_nsec_per_intr = 0; 457 458 /* 459 * apic_let_idle_redistribute can have the following values: 460 * 0 - If clock decremented it from 1 to 0, clock has to call redistribute. 461 * apic_redistribute_lock prevents multiple idle cpus from redistributing 462 */ 463 int apic_num_idle_redistributions = 0; 464 static int apic_let_idle_redistribute = 0; 465 static uint_t apic_nticks = 0; 466 static uint_t apic_skipped_redistribute = 0; 467 468 /* to gather intr data and redistribute */ 469 static void apic_redistribute_compute(void); 470 471 static uint_t last_count_read = 0; 472 static lock_t apic_gethrtime_lock; 473 volatile int apic_hrtime_stamp = 0; 474 volatile hrtime_t apic_nsec_since_boot = 0; 475 static uint_t apic_hertz_count, apic_nsec_per_tick; 476 static hrtime_t apic_nsec_max; 477 478 static hrtime_t apic_last_hrtime = 0; 479 int apic_hrtime_error = 0; 480 int apic_remote_hrterr = 0; 481 int apic_num_nmis = 0; 482 int apic_apic_error = 0; 483 int apic_num_apic_errors = 0; 484 int apic_num_cksum_errors = 0; 485 486 static uchar_t apic_io_id[MAX_IO_APIC]; 487 static uchar_t apic_io_ver[MAX_IO_APIC]; 488 static uchar_t apic_io_vectbase[MAX_IO_APIC]; 489 static uchar_t apic_io_vectend[MAX_IO_APIC]; 490 volatile int32_t *apicioadr[MAX_IO_APIC]; 491 /* 492 * apic_ioapic_lock protects the ioapics (reg select), the status, temp_bound 493 * and bound elements of cpus_info and the temp_cpu element of irq_struct 494 */ 495 lock_t apic_ioapic_lock; 496 497 /* 498 * apic_ioapic_reprogram_lock prevents a CPU from exiting 499 * apic_intr_exit before IOAPIC reprogramming information 500 * is collected. 501 */ 502 static lock_t apic_ioapic_reprogram_lock; 503 static int apic_io_max = 0; /* no. of i/o apics enabled */ 504 505 static struct apic_io_intr *apic_io_intrp = 0; 506 static struct apic_bus *apic_busp; 507 508 uchar_t apic_vector_to_irq[APIC_MAX_VECTOR+1]; 509 static uchar_t apic_resv_vector[MAXIPL+1]; 510 511 static char apic_level_intr[APIC_MAX_VECTOR+1]; 512 static int apic_error = 0; 513 /* values which apic_error can take. Not catastrophic, but may help debug */ 514 #define APIC_ERR_BOOT_EOI 0x1 515 #define APIC_ERR_GET_IPIVECT_FAIL 0x2 516 #define APIC_ERR_INVALID_INDEX 0x4 517 #define APIC_ERR_MARK_VECTOR_FAIL 0x8 518 #define APIC_ERR_APIC_ERROR 0x40000000 519 #define APIC_ERR_NMI 0x80000000 520 521 static int apic_cmos_ssb_set = 0; 522 523 static uint32_t eisa_level_intr_mask = 0; 524 /* At least MSB will be set if EISA bus */ 525 526 static int apic_pci_bus_total = 0; 527 static uchar_t apic_single_pci_busid = 0; 528 529 530 /* 531 * airq_mutex protects additions to the apic_irq_table - the first 532 * pointer and any airq_nexts off of that one. It also protects 533 * apic_max_device_irq & apic_min_device_irq. It also guarantees 534 * that share_id is unique as new ids are generated only when new 535 * irq_t structs are linked in. Once linked in the structs are never 536 * deleted. temp_cpu & mps_intr_index field indicate if it is programmed 537 * or allocated. Note that there is a slight gap between allocating in 538 * apic_introp_xlate and programming in addspl. 539 */ 540 kmutex_t airq_mutex; 541 apic_irq_t *apic_irq_table[APIC_MAX_VECTOR+1]; 542 int apic_max_device_irq = 0; 543 int apic_min_device_irq = APIC_MAX_VECTOR; 544 545 /* use to make sure only one cpu handles the nmi */ 546 static lock_t apic_nmi_lock; 547 /* use to make sure only one cpu handles the error interrupt */ 548 static lock_t apic_error_lock; 549 550 /* 551 * Following declarations are for revectoring; used when ISRs at different 552 * IPLs share an irq. 553 */ 554 static lock_t apic_revector_lock; 555 static int apic_revector_pending = 0; 556 static uchar_t *apic_oldvec_to_newvec; 557 static uchar_t *apic_newvec_to_oldvec; 558 559 /* Ensures that the IOAPIC-reprogramming timeout is not reentrant */ 560 static kmutex_t apic_reprogram_timeout_mutex; 561 562 static struct ioapic_reprogram_data { 563 int valid; /* This entry is valid */ 564 int bindcpu; /* The CPU to which the int will be bound */ 565 unsigned timeouts; /* # times the reprogram timeout was called */ 566 } apic_reprogram_info[APIC_MAX_VECTOR+1]; 567 /* 568 * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. apic_reprogram_info 569 * is indexed by IRQ number, NOT by vector number. 570 */ 571 572 573 /* 574 * The following added to identify a software poweroff method if available. 575 */ 576 577 static struct { 578 int poweroff_method; 579 char oem_id[APIC_MPS_OEM_ID_LEN + 1]; /* MAX + 1 for NULL */ 580 char prod_id[APIC_MPS_PROD_ID_LEN + 1]; /* MAX + 1 for NULL */ 581 } apic_mps_ids[] = { 582 { APIC_POWEROFF_VIA_RTC, "INTEL", "ALDER" }, /* 4300 */ 583 { APIC_POWEROFF_VIA_RTC, "NCR", "AMC" }, /* 4300 */ 584 { APIC_POWEROFF_VIA_ASPEN_BMC, "INTEL", "A450NX" }, /* 4400? */ 585 { APIC_POWEROFF_VIA_ASPEN_BMC, "INTEL", "AD450NX" }, /* 4400 */ 586 { APIC_POWEROFF_VIA_ASPEN_BMC, "INTEL", "AC450NX" }, /* 4400R */ 587 { APIC_POWEROFF_VIA_SITKA_BMC, "INTEL", "S450NX" }, /* S50 */ 588 { APIC_POWEROFF_VIA_SITKA_BMC, "INTEL", "SC450NX" } /* S50? */ 589 }; 590 591 int apic_poweroff_method = APIC_POWEROFF_NONE; 592 593 static struct { 594 uchar_t cntl; 595 uchar_t data; 596 } aspen_bmc[] = { 597 { CC_SMS_WR_START, 0x18 }, /* NetFn/LUN */ 598 { CC_SMS_WR_NEXT, 0x24 }, /* Cmd SET_WATCHDOG_TIMER */ 599 { CC_SMS_WR_NEXT, 0x84 }, /* DataByte 1: SMS/OS no log */ 600 { CC_SMS_WR_NEXT, 0x2 }, /* DataByte 2: Power Down */ 601 { CC_SMS_WR_NEXT, 0x0 }, /* DataByte 3: no pre-timeout */ 602 { CC_SMS_WR_NEXT, 0x0 }, /* DataByte 4: timer expir. */ 603 { CC_SMS_WR_NEXT, 0xa }, /* DataByte 5: init countdown */ 604 { CC_SMS_WR_END, 0x0 }, /* DataByte 6: init countdown */ 605 606 { CC_SMS_WR_START, 0x18 }, /* NetFn/LUN */ 607 { CC_SMS_WR_END, 0x22 } /* Cmd RESET_WATCHDOG_TIMER */ 608 }; 609 610 static struct { 611 int port; 612 uchar_t data; 613 } sitka_bmc[] = { 614 { SMS_COMMAND_REGISTER, SMS_WRITE_START }, 615 { SMS_DATA_REGISTER, 0x18 }, /* NetFn/LUN */ 616 { SMS_DATA_REGISTER, 0x24 }, /* Cmd SET_WATCHDOG_TIMER */ 617 { SMS_DATA_REGISTER, 0x84 }, /* DataByte 1: SMS/OS no log */ 618 { SMS_DATA_REGISTER, 0x2 }, /* DataByte 2: Power Down */ 619 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 3: no pre-timeout */ 620 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 4: timer expir. */ 621 { SMS_DATA_REGISTER, 0xa }, /* DataByte 5: init countdown */ 622 { SMS_COMMAND_REGISTER, SMS_WRITE_END }, 623 { SMS_DATA_REGISTER, 0x0 }, /* DataByte 6: init countdown */ 624 625 { SMS_COMMAND_REGISTER, SMS_WRITE_START }, 626 { SMS_DATA_REGISTER, 0x18 }, /* NetFn/LUN */ 627 { SMS_COMMAND_REGISTER, SMS_WRITE_END }, 628 { SMS_DATA_REGISTER, 0x22 } /* Cmd RESET_WATCHDOG_TIMER */ 629 }; 630 631 632 /* Patchable global variables. */ 633 int apic_kmdb_on_nmi = 0; /* 0 - no, 1 - yes enter kmdb */ 634 int apic_debug_mps_id = 0; /* 1 - print MPS ID strings */ 635 636 /* 637 * ACPI definitions 638 */ 639 /* _PIC method arguments */ 640 #define ACPI_PIC_MODE 0 641 #define ACPI_APIC_MODE 1 642 643 /* APIC error flags we care about */ 644 #define APIC_SEND_CS_ERROR 0x01 645 #define APIC_RECV_CS_ERROR 0x02 646 #define APIC_CS_ERRORS (APIC_SEND_CS_ERROR|APIC_RECV_CS_ERROR) 647 648 /* 649 * ACPI variables 650 */ 651 /* 1 = acpi is enabled & working, 0 = acpi is not enabled or not there */ 652 static int apic_enable_acpi = 0; 653 654 /* ACPI Multiple APIC Description Table ptr */ 655 static MULTIPLE_APIC_TABLE *acpi_mapic_dtp = NULL; 656 657 /* ACPI Interrupt Source Override Structure ptr */ 658 static MADT_INTERRUPT_OVERRIDE *acpi_isop = NULL; 659 static int acpi_iso_cnt = 0; 660 661 /* ACPI Non-maskable Interrupt Sources ptr */ 662 static MADT_NMI_SOURCE *acpi_nmi_sp = NULL; 663 static int acpi_nmi_scnt = 0; 664 static MADT_LOCAL_APIC_NMI *acpi_nmi_cp = NULL; 665 static int acpi_nmi_ccnt = 0; 666 667 /* 668 * extern declarations 669 */ 670 extern int intr_clear(void); 671 extern void intr_restore(uint_t); 672 #if defined(__amd64) 673 extern int intpri_use_cr8; 674 #endif /* __amd64 */ 675 676 extern int apic_pci_msi_enable_vector(dev_info_t *, int, int, 677 int, int, int); 678 extern apic_irq_t *apic_find_irq(dev_info_t *, struct intrspec *, int); 679 680 /* 681 * This is the loadable module wrapper 682 */ 683 684 int 685 _init(void) 686 { 687 if (apic_coarse_hrtime) 688 apic_ops.psm_gethrtime = &apic_gettime; 689 return (psm_mod_init(&apic_hdlp, &apic_psm_info)); 690 } 691 692 int 693 _fini(void) 694 { 695 return (psm_mod_fini(&apic_hdlp, &apic_psm_info)); 696 } 697 698 int 699 _info(struct modinfo *modinfop) 700 { 701 return (psm_mod_info(&apic_hdlp, &apic_psm_info, modinfop)); 702 } 703 704 /* 705 * Auto-configuration routines 706 */ 707 708 /* 709 * Look at MPSpec 1.4 (Intel Order # 242016-005) for details of what we do here 710 * May work with 1.1 - but not guaranteed. 711 * According to the MP Spec, the MP floating pointer structure 712 * will be searched in the order described below: 713 * 1. In the first kilobyte of Extended BIOS Data Area (EBDA) 714 * 2. Within the last kilobyte of system base memory 715 * 3. In the BIOS ROM address space between 0F0000h and 0FFFFh 716 * Once we find the right signature with proper checksum, we call 717 * either handle_defconf or parse_mpct to get all info necessary for 718 * subsequent operations. 719 */ 720 static int 721 apic_probe() 722 { 723 uint32_t mpct_addr, ebda_start = 0, base_mem_end; 724 caddr_t biosdatap; 725 caddr_t mpct; 726 caddr_t fptr; 727 int i, mpct_size, mapsize, retval = PSM_FAILURE; 728 ushort_t ebda_seg, base_mem_size; 729 struct apic_mpfps_hdr *fpsp; 730 struct apic_mp_cnf_hdr *hdrp; 731 int bypass_cpu_and_ioapics_in_mptables; 732 int acpi_user_options; 733 734 if (apic_forceload < 0) 735 return (retval); 736 737 /* Allow override for MADT-only mode */ 738 acpi_user_options = ddi_prop_get_int(DDI_DEV_T_ANY, ddi_root_node(), 0, 739 "acpi-user-options", 0); 740 apic_use_acpi_madt_only = ((acpi_user_options & ACPI_OUSER_MADT) != 0); 741 742 /* Allow apic_use_acpi to override MADT-only mode */ 743 if (!apic_use_acpi) 744 apic_use_acpi_madt_only = 0; 745 746 retval = acpi_probe(); 747 748 /* 749 * mapin the bios data area 40:0 750 * 40:13h - two-byte location reports the base memory size 751 * 40:0Eh - two-byte location for the exact starting address of 752 * the EBDA segment for EISA 753 */ 754 biosdatap = psm_map_phys(0x400, 0x20, PROT_READ); 755 if (!biosdatap) 756 return (retval); 757 fpsp = (struct apic_mpfps_hdr *)NULL; 758 mapsize = MPFPS_RAM_WIN_LEN; 759 /*LINTED: pointer cast may result in improper alignment */ 760 ebda_seg = *((ushort_t *)(biosdatap+0xe)); 761 /* check the 1k of EBDA */ 762 if (ebda_seg) { 763 ebda_start = ((uint32_t)ebda_seg) << 4; 764 fptr = psm_map_phys(ebda_start, MPFPS_RAM_WIN_LEN, PROT_READ); 765 if (fptr) { 766 if (!(fpsp = 767 apic_find_fps_sig(fptr, MPFPS_RAM_WIN_LEN))) 768 psm_unmap_phys(fptr, MPFPS_RAM_WIN_LEN); 769 } 770 } 771 /* If not in EBDA, check the last k of system base memory */ 772 if (!fpsp) { 773 /*LINTED: pointer cast may result in improper alignment */ 774 base_mem_size = *((ushort_t *)(biosdatap + 0x13)); 775 776 if (base_mem_size > 512) 777 base_mem_end = 639 * 1024; 778 else 779 base_mem_end = 511 * 1024; 780 /* if ebda == last k of base mem, skip to check BIOS ROM */ 781 if (base_mem_end != ebda_start) { 782 783 fptr = psm_map_phys(base_mem_end, MPFPS_RAM_WIN_LEN, 784 PROT_READ); 785 786 if (fptr) { 787 if (!(fpsp = apic_find_fps_sig(fptr, 788 MPFPS_RAM_WIN_LEN))) 789 psm_unmap_phys(fptr, MPFPS_RAM_WIN_LEN); 790 } 791 } 792 } 793 psm_unmap_phys(biosdatap, 0x20); 794 795 /* If still cannot find it, check the BIOS ROM space */ 796 if (!fpsp) { 797 mapsize = MPFPS_ROM_WIN_LEN; 798 fptr = psm_map_phys(MPFPS_ROM_WIN_START, 799 MPFPS_ROM_WIN_LEN, PROT_READ); 800 if (fptr) { 801 if (!(fpsp = 802 apic_find_fps_sig(fptr, MPFPS_ROM_WIN_LEN))) { 803 psm_unmap_phys(fptr, MPFPS_ROM_WIN_LEN); 804 return (retval); 805 } 806 } 807 } 808 809 if (apic_checksum((caddr_t)fpsp, fpsp->mpfps_length * 16) != 0) { 810 psm_unmap_phys(fptr, MPFPS_ROM_WIN_LEN); 811 return (retval); 812 } 813 814 apic_spec_rev = fpsp->mpfps_spec_rev; 815 if ((apic_spec_rev != 04) && (apic_spec_rev != 01)) { 816 psm_unmap_phys(fptr, MPFPS_ROM_WIN_LEN); 817 return (retval); 818 } 819 820 /* check IMCR is present or not */ 821 apic_imcrp = fpsp->mpfps_featinfo2 & MPFPS_FEATINFO2_IMCRP; 822 823 /* check default configuration (dual CPUs) */ 824 if ((apic_defconf = fpsp->mpfps_featinfo1) != 0) { 825 psm_unmap_phys(fptr, mapsize); 826 return (apic_handle_defconf()); 827 } 828 829 /* MP Configuration Table */ 830 mpct_addr = (uint32_t)(fpsp->mpfps_mpct_paddr); 831 832 psm_unmap_phys(fptr, mapsize); /* unmap floating ptr struct */ 833 834 /* 835 * Map in enough memory for the MP Configuration Table Header. 836 * Use this table to read the total length of the BIOS data and 837 * map in all the info 838 */ 839 /*LINTED: pointer cast may result in improper alignment */ 840 hdrp = (struct apic_mp_cnf_hdr *)psm_map_phys(mpct_addr, 841 sizeof (struct apic_mp_cnf_hdr), PROT_READ); 842 if (!hdrp) 843 return (retval); 844 845 /* check mp configuration table signature PCMP */ 846 if (hdrp->mpcnf_sig != 0x504d4350) { 847 psm_unmap_phys((caddr_t)hdrp, sizeof (struct apic_mp_cnf_hdr)); 848 return (retval); 849 } 850 mpct_size = (int)hdrp->mpcnf_tbl_length; 851 852 apic_set_pwroff_method_from_mpcnfhdr(hdrp); 853 854 psm_unmap_phys((caddr_t)hdrp, sizeof (struct apic_mp_cnf_hdr)); 855 856 if ((retval == PSM_SUCCESS) && !apic_use_acpi_madt_only) { 857 /* This is an ACPI machine No need for further checks */ 858 return (retval); 859 } 860 861 /* 862 * Map in the entries for this machine, ie. Processor 863 * Entry Tables, Bus Entry Tables, etc. 864 * They are in fixed order following one another 865 */ 866 mpct = psm_map_phys(mpct_addr, mpct_size, PROT_READ); 867 if (!mpct) 868 return (retval); 869 870 if (apic_checksum(mpct, mpct_size) != 0) 871 goto apic_fail1; 872 873 874 /*LINTED: pointer cast may result in improper alignment */ 875 hdrp = (struct apic_mp_cnf_hdr *)mpct; 876 /*LINTED: pointer cast may result in improper alignment */ 877 apicadr = (uint32_t *)psm_map_phys((uint32_t)hdrp->mpcnf_local_apic, 878 APIC_LOCAL_MEMLEN, PROT_READ | PROT_WRITE); 879 if (!apicadr) 880 goto apic_fail1; 881 882 /* Parse all information in the tables */ 883 bypass_cpu_and_ioapics_in_mptables = (retval == PSM_SUCCESS); 884 if (apic_parse_mpct(mpct, bypass_cpu_and_ioapics_in_mptables) == 885 PSM_SUCCESS) 886 return (PSM_SUCCESS); 887 888 for (i = 0; i < apic_io_max; i++) 889 psm_unmap_phys((caddr_t)apicioadr[i], APIC_IO_MEMLEN); 890 if (apic_cpus) 891 kmem_free(apic_cpus, sizeof (*apic_cpus) * apic_nproc); 892 if (apicadr) 893 psm_unmap_phys((caddr_t)apicadr, APIC_LOCAL_MEMLEN); 894 apic_fail1: 895 psm_unmap_phys(mpct, mpct_size); 896 return (retval); 897 } 898 899 static void 900 apic_set_pwroff_method_from_mpcnfhdr(struct apic_mp_cnf_hdr *hdrp) 901 { 902 int i; 903 904 for (i = 0; i < (sizeof (apic_mps_ids) / sizeof (apic_mps_ids[0])); 905 i++) { 906 if ((strncmp(hdrp->mpcnf_oem_str, apic_mps_ids[i].oem_id, 907 strlen(apic_mps_ids[i].oem_id)) == 0) && 908 (strncmp(hdrp->mpcnf_prod_str, apic_mps_ids[i].prod_id, 909 strlen(apic_mps_ids[i].prod_id)) == 0)) { 910 911 apic_poweroff_method = apic_mps_ids[i].poweroff_method; 912 break; 913 } 914 } 915 916 if (apic_debug_mps_id != 0) { 917 cmn_err(CE_CONT, "pcplusmp: MPS OEM ID = '%c%c%c%c%c%c%c%c'" 918 "Product ID = '%c%c%c%c%c%c%c%c%c%c%c%c'\n", 919 hdrp->mpcnf_oem_str[0], 920 hdrp->mpcnf_oem_str[1], 921 hdrp->mpcnf_oem_str[2], 922 hdrp->mpcnf_oem_str[3], 923 hdrp->mpcnf_oem_str[4], 924 hdrp->mpcnf_oem_str[5], 925 hdrp->mpcnf_oem_str[6], 926 hdrp->mpcnf_oem_str[7], 927 hdrp->mpcnf_prod_str[0], 928 hdrp->mpcnf_prod_str[1], 929 hdrp->mpcnf_prod_str[2], 930 hdrp->mpcnf_prod_str[3], 931 hdrp->mpcnf_prod_str[4], 932 hdrp->mpcnf_prod_str[5], 933 hdrp->mpcnf_prod_str[6], 934 hdrp->mpcnf_prod_str[7], 935 hdrp->mpcnf_prod_str[8], 936 hdrp->mpcnf_prod_str[9], 937 hdrp->mpcnf_prod_str[10], 938 hdrp->mpcnf_prod_str[11]); 939 } 940 } 941 942 static int 943 acpi_probe(void) 944 { 945 int i, id, intmax, ver, index, rv; 946 int acpi_verboseflags = 0; 947 int madt_seen, madt_size; 948 APIC_HEADER *ap; 949 MADT_PROCESSOR_APIC *mpa; 950 MADT_IO_APIC *mia; 951 MADT_IO_SAPIC *misa; 952 MADT_INTERRUPT_OVERRIDE *mio; 953 MADT_NMI_SOURCE *mns; 954 MADT_INTERRUPT_SOURCE *mis; 955 MADT_LOCAL_APIC_NMI *mlan; 956 MADT_ADDRESS_OVERRIDE *mao; 957 ACPI_OBJECT_LIST arglist; 958 ACPI_OBJECT arg; 959 int sci; 960 iflag_t sci_flags; 961 volatile int32_t *ioapic; 962 char local_ids[NCPU]; 963 char proc_ids[NCPU]; 964 uchar_t hid; 965 966 if (!apic_use_acpi) 967 return (PSM_FAILURE); 968 969 if (AcpiGetFirmwareTable(APIC_SIG, 1, ACPI_LOGICAL_ADDRESSING, 970 (ACPI_TABLE_HEADER **) &acpi_mapic_dtp) != AE_OK) 971 return (PSM_FAILURE); 972 973 apicadr = (uint32_t *)psm_map_phys( 974 (uint32_t)acpi_mapic_dtp->LocalApicAddress, 975 APIC_LOCAL_MEMLEN, PROT_READ | PROT_WRITE); 976 if (!apicadr) 977 return (PSM_FAILURE); 978 979 id = apicadr[APIC_LID_REG]; 980 local_ids[0] = (uchar_t)(((uint_t)id) >> 24); 981 apic_nproc = index = 1; 982 apic_io_max = 0; 983 984 ap = (APIC_HEADER *) (acpi_mapic_dtp + 1); 985 madt_size = acpi_mapic_dtp->Length; 986 madt_seen = sizeof (*acpi_mapic_dtp); 987 988 while (madt_seen < madt_size) { 989 switch (ap->Type) { 990 case APIC_PROCESSOR: 991 mpa = (MADT_PROCESSOR_APIC *) ap; 992 if (mpa->ProcessorEnabled) { 993 if (mpa->LocalApicId == local_ids[0]) 994 proc_ids[0] = mpa->ProcessorId; 995 else if (apic_nproc < NCPU) { 996 local_ids[index] = mpa->LocalApicId; 997 proc_ids[index] = mpa->ProcessorId; 998 index++; 999 apic_nproc++; 1000 } else 1001 cmn_err(CE_WARN, "pcplusmp: exceeded " 1002 "maximum no. of CPUs (= %d)", NCPU); 1003 } 1004 break; 1005 1006 case APIC_IO: 1007 mia = (MADT_IO_APIC *) ap; 1008 if (apic_io_max < MAX_IO_APIC) { 1009 apic_io_id[apic_io_max] = mia->IoApicId; 1010 apic_io_vectbase[apic_io_max] = 1011 mia->Interrupt; 1012 ioapic = apicioadr[apic_io_max] = 1013 (int32_t *)psm_map_phys( 1014 (uint32_t)mia->Address, 1015 APIC_IO_MEMLEN, PROT_READ | PROT_WRITE); 1016 if (!ioapic) 1017 goto cleanup; 1018 apic_io_max++; 1019 } 1020 break; 1021 1022 case APIC_XRUPT_OVERRIDE: 1023 mio = (MADT_INTERRUPT_OVERRIDE *) ap; 1024 if (acpi_isop == NULL) 1025 acpi_isop = mio; 1026 acpi_iso_cnt++; 1027 break; 1028 1029 case APIC_NMI: 1030 /* UNIMPLEMENTED */ 1031 mns = (MADT_NMI_SOURCE *) ap; 1032 if (acpi_nmi_sp == NULL) 1033 acpi_nmi_sp = mns; 1034 acpi_nmi_scnt++; 1035 1036 cmn_err(CE_NOTE, "!apic: nmi source: %d %d %d\n", 1037 mns->Interrupt, mns->Polarity, 1038 mns->TriggerMode); 1039 break; 1040 1041 case APIC_LOCAL_NMI: 1042 /* UNIMPLEMENTED */ 1043 mlan = (MADT_LOCAL_APIC_NMI *) ap; 1044 if (acpi_nmi_cp == NULL) 1045 acpi_nmi_cp = mlan; 1046 acpi_nmi_ccnt++; 1047 1048 cmn_err(CE_NOTE, "!apic: local nmi: %d %d %d %d\n", 1049 mlan->ProcessorId, mlan->Polarity, 1050 mlan->TriggerMode, mlan->Lint); 1051 break; 1052 1053 case APIC_ADDRESS_OVERRIDE: 1054 /* UNIMPLEMENTED */ 1055 mao = (MADT_ADDRESS_OVERRIDE *) ap; 1056 cmn_err(CE_NOTE, "!apic: address override: %lx\n", 1057 (long)mao->Address); 1058 break; 1059 1060 case APIC_IO_SAPIC: 1061 /* UNIMPLEMENTED */ 1062 misa = (MADT_IO_SAPIC *) ap; 1063 1064 cmn_err(CE_NOTE, "!apic: io sapic: %d %d %lx\n", 1065 misa->IoSapicId, misa->InterruptBase, 1066 (long)misa->Address); 1067 break; 1068 1069 case APIC_XRUPT_SOURCE: 1070 /* UNIMPLEMENTED */ 1071 mis = (MADT_INTERRUPT_SOURCE *) ap; 1072 1073 cmn_err(CE_NOTE, 1074 "!apic: irq source: %d %d %d %d %d %d %d\n", 1075 mis->ProcessorId, mis->ProcessorEid, 1076 mis->Interrupt, mis->Polarity, 1077 mis->TriggerMode, mis->InterruptType, 1078 mis->IoSapicVector); 1079 break; 1080 case APIC_RESERVED: 1081 default: 1082 goto cleanup; 1083 } 1084 1085 /* advance to next entry */ 1086 madt_seen += ap->Length; 1087 ap = (APIC_HEADER *)(((char *)ap) + ap->Length); 1088 } 1089 1090 if ((apic_cpus = kmem_zalloc(sizeof (*apic_cpus) * apic_nproc, 1091 KM_NOSLEEP)) == NULL) 1092 goto cleanup; 1093 1094 apic_cpumask = (1 << apic_nproc) - 1; 1095 1096 /* 1097 * ACPI doesn't provide the local apic ver, get it directly from the 1098 * local apic 1099 */ 1100 ver = apicadr[APIC_VERS_REG]; 1101 for (i = 0; i < apic_nproc; i++) { 1102 apic_cpus[i].aci_local_id = local_ids[i]; 1103 apic_cpus[i].aci_local_ver = (uchar_t)(ver & 0xFF); 1104 } 1105 for (i = 0; i < apic_io_max; i++) { 1106 ioapic = apicioadr[i]; 1107 1108 /* 1109 * need to check Sitka on the following acpi problem 1110 * On the Sitka, the ioapic's apic_id field isn't reporting 1111 * the actual io apic id. We have reported this problem 1112 * to Intel. Until they fix the problem, we will get the 1113 * actual id directly from the ioapic. 1114 */ 1115 ioapic[APIC_IO_REG] = APIC_ID_CMD; 1116 id = ioapic[APIC_IO_DATA]; 1117 hid = (uchar_t)(((uint_t)id) >> 24); 1118 1119 if (hid != apic_io_id[i]) { 1120 if (apic_io_id[i] == 0) 1121 apic_io_id[i] = hid; 1122 else { /* set ioapic id to whatever reported by ACPI */ 1123 id = ((int32_t)apic_io_id[i]) << 24; 1124 ioapic[APIC_IO_REG] = APIC_ID_CMD; 1125 ioapic[APIC_IO_DATA] = id; 1126 } 1127 } 1128 ioapic[APIC_IO_REG] = APIC_VERS_CMD; 1129 ver = ioapic[APIC_IO_DATA]; 1130 apic_io_ver[i] = (uchar_t)(ver & 0xff); 1131 intmax = (ver >> 16) & 0xff; 1132 apic_io_vectend[i] = apic_io_vectbase[i] + intmax; 1133 } 1134 1135 1136 /* 1137 * Process SCI configuration here 1138 * An error may be returned here if 1139 * acpi-user-options specifies legacy mode 1140 * (no SCI, no ACPI mode) 1141 */ 1142 if (acpica_get_sci(&sci, &sci_flags) != AE_OK) 1143 sci = -1; 1144 1145 /* 1146 * Now call acpi_init() to generate namespaces 1147 * If this fails, we don't attempt to use ACPI 1148 * even if we were able to get a MADT above 1149 */ 1150 if (acpica_init() != AE_OK) 1151 goto cleanup; 1152 1153 /* 1154 * Squirrel away the SCI and flags for later on 1155 * in apic_picinit() when we're ready 1156 */ 1157 apic_sci_vect = sci; 1158 apic_sci_flags = sci_flags; 1159 1160 if (apic_verbose & APIC_VERBOSE_IRQ_FLAG) 1161 acpi_verboseflags |= PSM_VERBOSE_IRQ_FLAG; 1162 1163 if (apic_verbose & APIC_VERBOSE_POWEROFF_FLAG) 1164 acpi_verboseflags |= PSM_VERBOSE_POWEROFF_FLAG; 1165 1166 if (apic_verbose & APIC_VERBOSE_POWEROFF_PAUSE_FLAG) 1167 acpi_verboseflags |= PSM_VERBOSE_POWEROFF_PAUSE_FLAG; 1168 1169 if (acpi_psm_init(apic_psm_info.p_mach_idstring, acpi_verboseflags) == 1170 ACPI_PSM_FAILURE) 1171 goto cleanup; 1172 1173 /* Enable ACPI APIC interrupt routing */ 1174 arglist.Count = 1; 1175 arglist.Pointer = &arg; 1176 arg.Type = ACPI_TYPE_INTEGER; 1177 arg.Integer.Value = ACPI_APIC_MODE; /* 1 */ 1178 rv = AcpiEvaluateObject(NULL, "\\_PIC", &arglist, NULL); 1179 if (rv == AE_OK) { 1180 build_reserved_irqlist((uchar_t *)apic_reserved_irqlist); 1181 apic_enable_acpi = 1; 1182 if (apic_use_acpi_madt_only) { 1183 cmn_err(CE_CONT, 1184 "?Using ACPI for CPU/IOAPIC information ONLY\n"); 1185 } 1186 return (PSM_SUCCESS); 1187 } 1188 /* if setting APIC mode failed above, we fall through to cleanup */ 1189 1190 cleanup: 1191 if (apicadr != NULL) { 1192 psm_unmap_phys((caddr_t)apicadr, APIC_LOCAL_MEMLEN); 1193 apicadr = NULL; 1194 } 1195 apic_nproc = 0; 1196 for (i = 0; i < apic_io_max; i++) { 1197 psm_unmap_phys((caddr_t)apicioadr[i], APIC_IO_MEMLEN); 1198 apicioadr[i] = NULL; 1199 } 1200 apic_io_max = 0; 1201 acpi_isop = NULL; 1202 acpi_iso_cnt = 0; 1203 acpi_nmi_sp = NULL; 1204 acpi_nmi_scnt = 0; 1205 acpi_nmi_cp = NULL; 1206 acpi_nmi_ccnt = 0; 1207 return (PSM_FAILURE); 1208 } 1209 1210 /* 1211 * Handle default configuration. Fill in reqd global variables & tables 1212 * Fill all details as MP table does not give any more info 1213 */ 1214 static int 1215 apic_handle_defconf() 1216 { 1217 uint_t lid; 1218 1219 /*LINTED: pointer cast may result in improper alignment */ 1220 apicioadr[0] = (int32_t *)psm_map_phys(APIC_IO_ADDR, 1221 APIC_IO_MEMLEN, PROT_READ | PROT_WRITE); 1222 /*LINTED: pointer cast may result in improper alignment */ 1223 apicadr = (uint32_t *)psm_map_phys(APIC_LOCAL_ADDR, 1224 APIC_LOCAL_MEMLEN, PROT_READ | PROT_WRITE); 1225 apic_cpus = (apic_cpus_info_t *) 1226 kmem_zalloc(sizeof (*apic_cpus) * 2, KM_NOSLEEP); 1227 if ((!apicadr) || (!apicioadr[0]) || (!apic_cpus)) 1228 goto apic_handle_defconf_fail; 1229 apic_cpumask = 3; 1230 apic_nproc = 2; 1231 lid = apicadr[APIC_LID_REG]; 1232 apic_cpus[0].aci_local_id = (uchar_t)(lid >> APIC_ID_BIT_OFFSET); 1233 /* 1234 * According to the PC+MP spec 1.1, the local ids 1235 * for the default configuration has to be 0 or 1 1236 */ 1237 if (apic_cpus[0].aci_local_id == 1) 1238 apic_cpus[1].aci_local_id = 0; 1239 else if (apic_cpus[0].aci_local_id == 0) 1240 apic_cpus[1].aci_local_id = 1; 1241 else 1242 goto apic_handle_defconf_fail; 1243 1244 apic_io_id[0] = 2; 1245 apic_io_max = 1; 1246 if (apic_defconf >= 5) { 1247 apic_cpus[0].aci_local_ver = APIC_INTEGRATED_VERS; 1248 apic_cpus[1].aci_local_ver = APIC_INTEGRATED_VERS; 1249 apic_io_ver[0] = APIC_INTEGRATED_VERS; 1250 } else { 1251 apic_cpus[0].aci_local_ver = 0; /* 82489 DX */ 1252 apic_cpus[1].aci_local_ver = 0; 1253 apic_io_ver[0] = 0; 1254 } 1255 if (apic_defconf == 2 || apic_defconf == 3 || apic_defconf == 6) 1256 eisa_level_intr_mask = (inb(EISA_LEVEL_CNTL + 1) << 8) | 1257 inb(EISA_LEVEL_CNTL) | ((uint_t)INT32_MAX + 1); 1258 return (PSM_SUCCESS); 1259 1260 apic_handle_defconf_fail: 1261 if (apic_cpus) 1262 kmem_free(apic_cpus, sizeof (*apic_cpus) * 2); 1263 if (apicadr) 1264 psm_unmap_phys((caddr_t)apicadr, APIC_LOCAL_MEMLEN); 1265 if (apicioadr[0]) 1266 psm_unmap_phys((caddr_t)apicioadr[0], APIC_IO_MEMLEN); 1267 return (PSM_FAILURE); 1268 } 1269 1270 /* Parse the entries in MP configuration table and collect info that we need */ 1271 static int 1272 apic_parse_mpct(caddr_t mpct, int bypass_cpus_and_ioapics) 1273 { 1274 struct apic_procent *procp; 1275 struct apic_bus *busp; 1276 struct apic_io_entry *ioapicp; 1277 struct apic_io_intr *intrp; 1278 volatile int32_t *ioapic; 1279 uint_t lid; 1280 int id; 1281 uchar_t hid; 1282 1283 /*LINTED: pointer cast may result in improper alignment */ 1284 procp = (struct apic_procent *)(mpct + sizeof (struct apic_mp_cnf_hdr)); 1285 1286 /* No need to count cpu entries if we won't use them */ 1287 if (!bypass_cpus_and_ioapics) { 1288 1289 /* Find max # of CPUS and allocate structure accordingly */ 1290 apic_nproc = 0; 1291 while (procp->proc_entry == APIC_CPU_ENTRY) { 1292 if (procp->proc_cpuflags & CPUFLAGS_EN) { 1293 apic_nproc++; 1294 } 1295 procp++; 1296 } 1297 if (apic_nproc > NCPU) 1298 cmn_err(CE_WARN, "pcplusmp: exceeded " 1299 "maximum no. of CPUs (= %d)", NCPU); 1300 if (!apic_nproc || !(apic_cpus = (apic_cpus_info_t *) 1301 kmem_zalloc(sizeof (*apic_cpus)*apic_nproc, KM_NOSLEEP))) 1302 return (PSM_FAILURE); 1303 } 1304 1305 /*LINTED: pointer cast may result in improper alignment */ 1306 procp = (struct apic_procent *)(mpct + sizeof (struct apic_mp_cnf_hdr)); 1307 1308 /* 1309 * start with index 1 as 0 needs to be filled in with Boot CPU, but 1310 * if we're bypassing this information, it has already been filled 1311 * in by acpi_probe(), so don't overwrite it. 1312 */ 1313 if (!bypass_cpus_and_ioapics) 1314 apic_nproc = 1; 1315 1316 while (procp->proc_entry == APIC_CPU_ENTRY) { 1317 /* check whether the cpu exists or not */ 1318 if (!bypass_cpus_and_ioapics && 1319 procp->proc_cpuflags & CPUFLAGS_EN) { 1320 if (procp->proc_cpuflags & CPUFLAGS_BP) { /* Boot CPU */ 1321 lid = apicadr[APIC_LID_REG]; 1322 apic_cpus[0].aci_local_id = procp->proc_apicid; 1323 if (apic_cpus[0].aci_local_id != 1324 (uchar_t)(lid >> APIC_ID_BIT_OFFSET)) { 1325 return (PSM_FAILURE); 1326 } 1327 apic_cpus[0].aci_local_ver = 1328 procp->proc_version; 1329 } else { 1330 1331 apic_cpus[apic_nproc].aci_local_id = 1332 procp->proc_apicid; 1333 apic_cpus[apic_nproc].aci_local_ver = 1334 procp->proc_version; 1335 apic_nproc++; 1336 1337 } 1338 } 1339 procp++; 1340 } 1341 1342 if (!bypass_cpus_and_ioapics) { 1343 /* convert the number of processors into a cpumask */ 1344 apic_cpumask = (1 << apic_nproc) - 1; 1345 } 1346 1347 /* 1348 * Save start of bus entries for later use. 1349 * Get EISA level cntrl if EISA bus is present. 1350 * Also get the CPI bus id for single CPI bus case 1351 */ 1352 apic_busp = busp = (struct apic_bus *)procp; 1353 while (busp->bus_entry == APIC_BUS_ENTRY) { 1354 lid = apic_find_bus_type((char *)&busp->bus_str1); 1355 if (lid == BUS_EISA) { 1356 eisa_level_intr_mask = (inb(EISA_LEVEL_CNTL + 1) << 8) | 1357 inb(EISA_LEVEL_CNTL) | ((uint_t)INT32_MAX + 1); 1358 } else if (lid == BUS_PCI) { 1359 /* 1360 * apic_single_pci_busid will be used only if 1361 * apic_pic_bus_total is equal to 1 1362 */ 1363 apic_pci_bus_total++; 1364 apic_single_pci_busid = busp->bus_id; 1365 } 1366 busp++; 1367 } 1368 1369 ioapicp = (struct apic_io_entry *)busp; 1370 1371 if (!bypass_cpus_and_ioapics) 1372 apic_io_max = 0; 1373 do { 1374 if (!bypass_cpus_and_ioapics && apic_io_max < MAX_IO_APIC) { 1375 if (ioapicp->io_flags & IOAPIC_FLAGS_EN) { 1376 apic_io_id[apic_io_max] = ioapicp->io_apicid; 1377 apic_io_ver[apic_io_max] = ioapicp->io_version; 1378 /*LINTED: pointer cast may result in improper alignment */ 1379 apicioadr[apic_io_max] = 1380 (int32_t *)psm_map_phys( 1381 (uint32_t)ioapicp->io_apic_addr, 1382 APIC_IO_MEMLEN, PROT_READ | PROT_WRITE); 1383 1384 if (!apicioadr[apic_io_max]) 1385 return (PSM_FAILURE); 1386 1387 ioapic = apicioadr[apic_io_max]; 1388 ioapic[APIC_IO_REG] = APIC_ID_CMD; 1389 id = ioapic[APIC_IO_DATA]; 1390 hid = (uchar_t)(((uint_t)id) >> 24); 1391 1392 if (hid != apic_io_id[apic_io_max]) { 1393 if (apic_io_id[apic_io_max] == 0) 1394 apic_io_id[apic_io_max] = hid; 1395 else { 1396 /* 1397 * set ioapic id to whatever 1398 * reported by MPS 1399 * 1400 * may not need to set index 1401 * again ??? 1402 * take it out and try 1403 */ 1404 1405 id = ((int32_t) 1406 apic_io_id[apic_io_max]) << 1407 24; 1408 1409 ioapic[APIC_IO_REG] = 1410 APIC_ID_CMD; 1411 1412 ioapic[APIC_IO_DATA] = id; 1413 1414 } 1415 } 1416 apic_io_max++; 1417 } 1418 } 1419 ioapicp++; 1420 } while (ioapicp->io_entry == APIC_IO_ENTRY); 1421 1422 apic_io_intrp = (struct apic_io_intr *)ioapicp; 1423 1424 intrp = apic_io_intrp; 1425 while (intrp->intr_entry == APIC_IO_INTR_ENTRY) { 1426 if ((intrp->intr_irq > APIC_MAX_ISA_IRQ) || 1427 (apic_find_bus(intrp->intr_busid) == BUS_PCI)) { 1428 apic_irq_translate = 1; 1429 break; 1430 } 1431 intrp++; 1432 } 1433 1434 return (PSM_SUCCESS); 1435 } 1436 1437 static struct apic_mpfps_hdr * 1438 apic_find_fps_sig(caddr_t cptr, int len) 1439 { 1440 int i; 1441 1442 /* Look for the pattern "_MP_" */ 1443 for (i = 0; i < len; i += 16) { 1444 if ((*(cptr+i) == '_') && 1445 (*(cptr+i+1) == 'M') && 1446 (*(cptr+i+2) == 'P') && 1447 (*(cptr+i+3) == '_')) 1448 /*LINTED: pointer cast may result in improper alignment */ 1449 return ((struct apic_mpfps_hdr *)(cptr + i)); 1450 } 1451 return (NULL); 1452 } 1453 1454 static int 1455 apic_checksum(caddr_t bptr, int len) 1456 { 1457 int i; 1458 uchar_t cksum; 1459 1460 cksum = 0; 1461 for (i = 0; i < len; i++) 1462 cksum += *bptr++; 1463 return ((int)cksum); 1464 } 1465 1466 1467 /* 1468 * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable 1469 * are also set to NULL. vector->irq is set to a value which cannot map 1470 * to a real irq to show that it is free. 1471 */ 1472 void 1473 apic_init() 1474 { 1475 int i; 1476 int *iptr; 1477 1478 int j = 1; 1479 apic_ipltopri[0] = APIC_VECTOR_PER_IPL; /* leave 0 for idle */ 1480 for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) { 1481 if ((i < ((APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL) - 1)) && 1482 (apic_vectortoipl[i + 1] == apic_vectortoipl[i])) 1483 /* get to highest vector at the same ipl */ 1484 continue; 1485 for (; j <= apic_vectortoipl[i]; j++) { 1486 apic_ipltopri[j] = (i << APIC_IPL_SHIFT) + 1487 APIC_BASE_VECT; 1488 } 1489 } 1490 for (; j < MAXIPL + 1; j++) 1491 /* fill up any empty ipltopri slots */ 1492 apic_ipltopri[j] = (i << APIC_IPL_SHIFT) + APIC_BASE_VECT; 1493 1494 /* cpu 0 is always up */ 1495 apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE; 1496 1497 iptr = (int *)&apic_irq_table[0]; 1498 for (i = 0; i <= APIC_MAX_VECTOR; i++) { 1499 apic_level_intr[i] = 0; 1500 *iptr++ = NULL; 1501 apic_vector_to_irq[i] = APIC_RESV_IRQ; 1502 apic_reprogram_info[i].valid = 0; 1503 apic_reprogram_info[i].bindcpu = 0; 1504 apic_reprogram_info[i].timeouts = 0; 1505 } 1506 1507 /* 1508 * Allocate a dummy irq table entry for the reserved entry. 1509 * This takes care of the race between removing an irq and 1510 * clock detecting a CPU in that irq during interrupt load 1511 * sampling. 1512 */ 1513 apic_irq_table[APIC_RESV_IRQ] = 1514 kmem_zalloc(sizeof (apic_irq_t), KM_NOSLEEP); 1515 1516 mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL); 1517 mutex_init(&apic_reprogram_timeout_mutex, NULL, MUTEX_DEFAULT, NULL); 1518 #if defined(__amd64) 1519 /* 1520 * Make cpu-specific interrupt info point to cr8pri vector 1521 */ 1522 for (i = 0; i <= MAXIPL; i++) 1523 apic_cr8pri[i] = apic_ipltopri[i] >> APIC_IPL_SHIFT; 1524 CPU->cpu_pri_data = apic_cr8pri; 1525 intpri_use_cr8 = 1; 1526 #endif /* __amd64 */ 1527 } 1528 1529 /* 1530 * handler for APIC Error interrupt. Just print a warning and continue 1531 */ 1532 static int 1533 apic_error_intr() 1534 { 1535 uint_t error0, error1, error; 1536 uint_t i; 1537 1538 /* 1539 * We need to write before read as per 7.4.17 of system prog manual. 1540 * We do both and or the results to be safe 1541 */ 1542 error0 = apicadr[APIC_ERROR_STATUS]; 1543 apicadr[APIC_ERROR_STATUS] = 0; 1544 error1 = apicadr[APIC_ERROR_STATUS]; 1545 error = error0 | error1; 1546 1547 /* 1548 * Prevent more than 1 CPU from handling error interrupt causing 1549 * double printing (interleave of characters from multiple 1550 * CPU's when using prom_printf) 1551 */ 1552 if (lock_try(&apic_error_lock) == 0) 1553 return (error ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 1554 if (error) { 1555 #if DEBUG 1556 if (apic_debug) 1557 debug_enter("pcplusmp: APIC Error interrupt received"); 1558 #endif /* DEBUG */ 1559 if (apic_panic_on_apic_error) 1560 cmn_err(CE_PANIC, 1561 "APIC Error interrupt on CPU %d. Status = %x\n", 1562 psm_get_cpu_id(), error); 1563 else { 1564 if ((error & ~APIC_CS_ERRORS) == 0) { 1565 /* cksum error only */ 1566 apic_error |= APIC_ERR_APIC_ERROR; 1567 apic_apic_error |= error; 1568 apic_num_apic_errors++; 1569 apic_num_cksum_errors++; 1570 } else { 1571 /* 1572 * prom_printf is the best shot we have of 1573 * something which is problem free from 1574 * high level/NMI type of interrupts 1575 */ 1576 prom_printf("APIC Error interrupt on CPU %d. " 1577 "Status 0 = %x, Status 1 = %x\n", 1578 psm_get_cpu_id(), error0, error1); 1579 apic_error |= APIC_ERR_APIC_ERROR; 1580 apic_apic_error |= error; 1581 apic_num_apic_errors++; 1582 for (i = 0; i < apic_error_display_delay; i++) { 1583 tenmicrosec(); 1584 } 1585 /* 1586 * provide more delay next time limited to 1587 * roughly 1 clock tick time 1588 */ 1589 if (apic_error_display_delay < 500) 1590 apic_error_display_delay *= 2; 1591 } 1592 } 1593 lock_clear(&apic_error_lock); 1594 return (DDI_INTR_CLAIMED); 1595 } else { 1596 lock_clear(&apic_error_lock); 1597 return (DDI_INTR_UNCLAIMED); 1598 } 1599 /* NOTREACHED */ 1600 } 1601 1602 /* 1603 * Turn off the mask bit in the performance counter Local Vector Table entry. 1604 */ 1605 static void 1606 apic_cpcovf_mask_clear(void) 1607 { 1608 apicadr[APIC_PCINT_VECT] &= ~APIC_LVT_MASK; 1609 } 1610 1611 static void 1612 apic_init_intr() 1613 { 1614 processorid_t cpun = psm_get_cpu_id(); 1615 1616 #if defined(__amd64) 1617 setcr8((ulong_t)(APIC_MASK_ALL >> APIC_IPL_SHIFT)); 1618 #else 1619 apicadr[APIC_TASK_REG] = APIC_MASK_ALL; 1620 #endif 1621 1622 if (apic_flat_model) 1623 apicadr[APIC_FORMAT_REG] = APIC_FLAT_MODEL; 1624 else 1625 apicadr[APIC_FORMAT_REG] = APIC_CLUSTER_MODEL; 1626 apicadr[APIC_DEST_REG] = AV_HIGH_ORDER >> cpun; 1627 1628 /* need to enable APIC before unmasking NMI */ 1629 apicadr[APIC_SPUR_INT_REG] = AV_UNIT_ENABLE | APIC_SPUR_INTR; 1630 1631 apicadr[APIC_LOCAL_TIMER] = AV_MASK; 1632 apicadr[APIC_INT_VECT0] = AV_MASK; /* local intr reg 0 */ 1633 apicadr[APIC_INT_VECT1] = AV_NMI; /* enable NMI */ 1634 1635 if (apic_cpus[cpun].aci_local_ver < APIC_INTEGRATED_VERS) 1636 return; 1637 1638 /* Enable performance counter overflow interrupt */ 1639 1640 if ((x86_feature & X86_MSR) != X86_MSR) 1641 apic_enable_cpcovf_intr = 0; 1642 if (apic_enable_cpcovf_intr) { 1643 if (apic_cpcovf_vect == 0) { 1644 int ipl = APIC_PCINT_IPL; 1645 int irq = apic_get_ipivect(ipl, -1); 1646 1647 ASSERT(irq != -1); 1648 apic_cpcovf_vect = apic_irq_table[irq]->airq_vector; 1649 ASSERT(apic_cpcovf_vect); 1650 (void) add_avintr(NULL, ipl, 1651 (avfunc)kcpc_hw_overflow_intr, 1652 "apic pcint", irq, NULL, NULL, NULL); 1653 kcpc_hw_overflow_intr_installed = 1; 1654 kcpc_hw_enable_cpc_intr = apic_cpcovf_mask_clear; 1655 } 1656 apicadr[APIC_PCINT_VECT] = apic_cpcovf_vect; 1657 } 1658 1659 /* Enable error interrupt */ 1660 1661 if (apic_enable_error_intr) { 1662 if (apic_errvect == 0) { 1663 int ipl = 0xf; /* get highest priority intr */ 1664 int irq = apic_get_ipivect(ipl, -1); 1665 1666 ASSERT(irq != -1); 1667 apic_errvect = apic_irq_table[irq]->airq_vector; 1668 ASSERT(apic_errvect); 1669 /* 1670 * Not PSMI compliant, but we are going to merge 1671 * with ON anyway 1672 */ 1673 (void) add_avintr((void *)NULL, ipl, 1674 (avfunc)apic_error_intr, "apic error intr", 1675 irq, NULL, NULL, NULL); 1676 } 1677 apicadr[APIC_ERR_VECT] = apic_errvect; 1678 apicadr[APIC_ERROR_STATUS] = 0; 1679 apicadr[APIC_ERROR_STATUS] = 0; 1680 } 1681 } 1682 1683 static void 1684 apic_disable_local_apic() 1685 { 1686 apicadr[APIC_TASK_REG] = APIC_MASK_ALL; 1687 apicadr[APIC_LOCAL_TIMER] = AV_MASK; 1688 apicadr[APIC_INT_VECT0] = AV_MASK; /* local intr reg 0 */ 1689 apicadr[APIC_INT_VECT1] = AV_MASK; /* disable NMI */ 1690 apicadr[APIC_ERR_VECT] = AV_MASK; /* and error interrupt */ 1691 apicadr[APIC_PCINT_VECT] = AV_MASK; /* and perf counter intr */ 1692 apicadr[APIC_SPUR_INT_REG] = APIC_SPUR_INTR; 1693 } 1694 1695 static void 1696 apic_picinit(void) 1697 { 1698 int i, j; 1699 uint_t isr; 1700 volatile int32_t *ioapic; 1701 apic_irq_t *irqptr; 1702 struct intrspec ispec; 1703 1704 /* 1705 * On UniSys Model 6520, the BIOS leaves vector 0x20 isr 1706 * bit on without clearing it with EOI. Since softint 1707 * uses vector 0x20 to interrupt itself, so softint will 1708 * not work on this machine. In order to fix this problem 1709 * a check is made to verify all the isr bits are clear. 1710 * If not, EOIs are issued to clear the bits. 1711 */ 1712 for (i = 7; i >= 1; i--) { 1713 if ((isr = apicadr[APIC_ISR_REG + (i * 4)]) != 0) 1714 for (j = 0; ((j < 32) && (isr != 0)); j++) 1715 if (isr & (1 << j)) { 1716 apicadr[APIC_EOI_REG] = 0; 1717 isr &= ~(1 << j); 1718 apic_error |= APIC_ERR_BOOT_EOI; 1719 } 1720 } 1721 1722 /* set a flag so we know we have run apic_picinit() */ 1723 apic_flag = 1; 1724 LOCK_INIT_CLEAR(&apic_gethrtime_lock); 1725 LOCK_INIT_CLEAR(&apic_ioapic_lock); 1726 LOCK_INIT_CLEAR(&apic_revector_lock); 1727 LOCK_INIT_CLEAR(&apic_ioapic_reprogram_lock); 1728 LOCK_INIT_CLEAR(&apic_error_lock); 1729 1730 picsetup(); /* initialise the 8259 */ 1731 1732 /* add nmi handler - least priority nmi handler */ 1733 LOCK_INIT_CLEAR(&apic_nmi_lock); 1734 1735 if (!psm_add_nmintr(0, (avfunc) apic_nmi_intr, 1736 "pcplusmp NMI handler", (caddr_t)NULL)) 1737 cmn_err(CE_WARN, "pcplusmp: Unable to add nmi handler"); 1738 1739 apic_init_intr(); 1740 1741 /* enable apic mode if imcr present */ 1742 if (apic_imcrp) { 1743 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT); 1744 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_APIC); 1745 } 1746 1747 /* mask interrupt vectors */ 1748 for (j = 0; j < apic_io_max; j++) { 1749 int intin_max; 1750 ioapic = apicioadr[j]; 1751 ioapic[APIC_IO_REG] = APIC_VERS_CMD; 1752 /* Bits 23-16 define the maximum redirection entries */ 1753 intin_max = (ioapic[APIC_IO_DATA] >> 16) & 0xff; 1754 for (i = 0; i < intin_max; i++) { 1755 ioapic[APIC_IO_REG] = APIC_RDT_CMD + 2 * i; 1756 ioapic[APIC_IO_DATA] = AV_MASK; 1757 } 1758 } 1759 1760 /* 1761 * Hack alert: deal with ACPI SCI interrupt chicken/egg here 1762 */ 1763 if (apic_sci_vect > 0) { 1764 /* 1765 * acpica has already done add_avintr(); we just 1766 * to finish the job by mimicing translate_irq() 1767 * 1768 * Fake up an intrspec and setup the tables 1769 */ 1770 ispec.intrspec_vec = apic_sci_vect; 1771 ispec.intrspec_pri = SCI_IPL; 1772 1773 if (apic_setup_irq_table(NULL, apic_sci_vect, NULL, 1774 &ispec, &apic_sci_flags, DDI_INTR_TYPE_FIXED) < 0) { 1775 cmn_err(CE_WARN, "!apic: SCI setup failed"); 1776 return; 1777 } 1778 irqptr = apic_irq_table[apic_sci_vect]; 1779 1780 /* Program I/O APIC */ 1781 (void) apic_setup_io_intr(irqptr, apic_sci_vect); 1782 } 1783 } 1784 1785 1786 static void 1787 apic_cpu_start(processorid_t cpun, caddr_t rm_code) 1788 { 1789 int loop_count; 1790 uint32_t vector; 1791 uint_t cpu_id, iflag; 1792 1793 cpu_id = apic_cpus[cpun].aci_local_id; 1794 1795 apic_cmos_ssb_set = 1; 1796 1797 /* 1798 * Interrupts on BSP cpu will be disabled during these startup 1799 * steps in order to avoid unwanted side effects from 1800 * executing interrupt handlers on a problematic BIOS. 1801 */ 1802 1803 iflag = intr_clear(); 1804 outb(CMOS_ADDR, SSB); 1805 outb(CMOS_DATA, BIOS_SHUTDOWN); 1806 1807 while (get_apic_cmd1() & AV_PENDING) 1808 apic_ret(); 1809 1810 /* for integrated - make sure there is one INIT IPI in buffer */ 1811 /* for external - it will wake up the cpu */ 1812 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET; 1813 apicadr[APIC_INT_CMD1] = AV_ASSERT | AV_RESET; 1814 1815 /* If only 1 CPU is installed, PENDING bit will not go low */ 1816 for (loop_count = 0x1000; loop_count; loop_count--) 1817 if (get_apic_cmd1() & AV_PENDING) 1818 apic_ret(); 1819 else 1820 break; 1821 1822 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET; 1823 apicadr[APIC_INT_CMD1] = AV_DEASSERT | AV_RESET; 1824 1825 drv_usecwait(20000); /* 20 milli sec */ 1826 1827 if (apic_cpus[cpun].aci_local_ver >= APIC_INTEGRATED_VERS) { 1828 /* integrated apic */ 1829 1830 rm_code = (caddr_t)(uintptr_t)rm_platter_pa; 1831 vector = (rm_platter_pa >> MMU_PAGESHIFT) & 1832 (APIC_VECTOR_MASK | APIC_IPL_MASK); 1833 1834 /* to offset the INIT IPI queue up in the buffer */ 1835 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET; 1836 apicadr[APIC_INT_CMD1] = vector | AV_STARTUP; 1837 1838 drv_usecwait(200); /* 20 micro sec */ 1839 1840 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET; 1841 apicadr[APIC_INT_CMD1] = vector | AV_STARTUP; 1842 1843 drv_usecwait(200); /* 20 micro sec */ 1844 } 1845 intr_restore(iflag); 1846 } 1847 1848 1849 #ifdef DEBUG 1850 int apic_break_on_cpu = 9; 1851 int apic_stretch_interrupts = 0; 1852 int apic_stretch_ISR = 1 << 3; /* IPL of 3 matches nothing now */ 1853 1854 void 1855 apic_break() 1856 { 1857 } 1858 #endif /* DEBUG */ 1859 1860 /* 1861 * platform_intr_enter 1862 * 1863 * Called at the beginning of the interrupt service routine to 1864 * mask all level equal to and below the interrupt priority 1865 * of the interrupting vector. An EOI should be given to 1866 * the interrupt controller to enable other HW interrupts. 1867 * 1868 * Return -1 for spurious interrupts 1869 * 1870 */ 1871 /*ARGSUSED*/ 1872 static int 1873 apic_intr_enter(int ipl, int *vectorp) 1874 { 1875 uchar_t vector; 1876 int nipl; 1877 int irq, iflag; 1878 apic_cpus_info_t *cpu_infop; 1879 1880 /* 1881 * The real vector programmed in APIC is *vectorp + 0x20 1882 * But, cmnint code subtracts 0x20 before pushing it. 1883 * Hence APIC_BASE_VECT is 0x20. 1884 */ 1885 1886 vector = (uchar_t)*vectorp; 1887 1888 /* if interrupted by the clock, increment apic_nsec_since_boot */ 1889 if (vector == apic_clkvect) { 1890 if (!apic_oneshot) { 1891 /* NOTE: this is not MT aware */ 1892 apic_hrtime_stamp++; 1893 apic_nsec_since_boot += apic_nsec_per_intr; 1894 apic_hrtime_stamp++; 1895 last_count_read = apic_hertz_count; 1896 apic_redistribute_compute(); 1897 } 1898 1899 /* We will avoid all the book keeping overhead for clock */ 1900 nipl = apic_vectortoipl[vector >> APIC_IPL_SHIFT]; 1901 #if defined(__amd64) 1902 setcr8((ulong_t)apic_cr8pri[nipl]); 1903 #else 1904 apicadr[APIC_TASK_REG] = apic_ipltopri[nipl]; 1905 #endif 1906 *vectorp = apic_vector_to_irq[vector + APIC_BASE_VECT]; 1907 apicadr[APIC_EOI_REG] = 0; 1908 return (nipl); 1909 } 1910 1911 cpu_infop = &apic_cpus[psm_get_cpu_id()]; 1912 1913 if (vector == (APIC_SPUR_INTR - APIC_BASE_VECT)) { 1914 cpu_infop->aci_spur_cnt++; 1915 return (APIC_INT_SPURIOUS); 1916 } 1917 1918 /* Check if the vector we got is really what we need */ 1919 if (apic_revector_pending) { 1920 /* 1921 * Disable interrupts for the duration of 1922 * the vector translation to prevent a self-race for 1923 * the apic_revector_lock. This cannot be done 1924 * in apic_xlate_vector because it is recursive and 1925 * we want the vector translation to be atomic with 1926 * respect to other (higher-priority) interrupts. 1927 */ 1928 iflag = intr_clear(); 1929 vector = apic_xlate_vector(vector + APIC_BASE_VECT) - 1930 APIC_BASE_VECT; 1931 intr_restore(iflag); 1932 } 1933 1934 nipl = apic_vectortoipl[vector >> APIC_IPL_SHIFT]; 1935 *vectorp = irq = apic_vector_to_irq[vector + APIC_BASE_VECT]; 1936 1937 #if defined(__amd64) 1938 setcr8((ulong_t)apic_cr8pri[nipl]); 1939 #else 1940 apicadr[APIC_TASK_REG] = apic_ipltopri[nipl]; 1941 #endif 1942 1943 cpu_infop->aci_current[nipl] = (uchar_t)irq; 1944 cpu_infop->aci_curipl = (uchar_t)nipl; 1945 cpu_infop->aci_ISR_in_progress |= 1 << nipl; 1946 1947 /* 1948 * apic_level_intr could have been assimilated into the irq struct. 1949 * but, having it as a character array is more efficient in terms of 1950 * cache usage. So, we leave it as is. 1951 */ 1952 if (!apic_level_intr[irq]) 1953 apicadr[APIC_EOI_REG] = 0; 1954 1955 #ifdef DEBUG 1956 APIC_DEBUG_BUF_PUT(vector); 1957 APIC_DEBUG_BUF_PUT(irq); 1958 APIC_DEBUG_BUF_PUT(nipl); 1959 APIC_DEBUG_BUF_PUT(psm_get_cpu_id()); 1960 if ((apic_stretch_interrupts) && (apic_stretch_ISR & (1 << nipl))) 1961 drv_usecwait(apic_stretch_interrupts); 1962 1963 if (apic_break_on_cpu == psm_get_cpu_id()) 1964 apic_break(); 1965 #endif /* DEBUG */ 1966 return (nipl); 1967 } 1968 1969 static void 1970 apic_intr_exit(int prev_ipl, int irq) 1971 { 1972 apic_cpus_info_t *cpu_infop; 1973 1974 #if defined(__amd64) 1975 setcr8((ulong_t)apic_cr8pri[prev_ipl]); 1976 #else 1977 apicadr[APIC_TASK_REG] = apic_ipltopri[prev_ipl]; 1978 #endif 1979 1980 cpu_infop = &apic_cpus[psm_get_cpu_id()]; 1981 if (apic_level_intr[irq]) 1982 apicadr[APIC_EOI_REG] = 0; 1983 1984 cpu_infop->aci_curipl = (uchar_t)prev_ipl; 1985 /* ISR above current pri could not be in progress */ 1986 cpu_infop->aci_ISR_in_progress &= (2 << prev_ipl) - 1; 1987 } 1988 1989 /* 1990 * Mask all interrupts below or equal to the given IPL 1991 */ 1992 static void 1993 apic_setspl(int ipl) 1994 { 1995 1996 #if defined(__amd64) 1997 setcr8((ulong_t)apic_cr8pri[ipl]); 1998 #else 1999 apicadr[APIC_TASK_REG] = apic_ipltopri[ipl]; 2000 #endif 2001 2002 /* interrupts at ipl above this cannot be in progress */ 2003 apic_cpus[psm_get_cpu_id()].aci_ISR_in_progress &= (2 << ipl) - 1; 2004 /* 2005 * this is a patch fix for the ALR QSMP P5 machine, so that interrupts 2006 * have enough time to come in before the priority is raised again 2007 * during the idle() loop. 2008 */ 2009 if (apic_setspl_delay) 2010 (void) get_apic_pri(); 2011 } 2012 2013 /* 2014 * trigger a software interrupt at the given IPL 2015 */ 2016 static void 2017 apic_set_softintr(int ipl) 2018 { 2019 int vector; 2020 uint_t flag; 2021 2022 vector = apic_resv_vector[ipl]; 2023 2024 flag = intr_clear(); 2025 2026 while (get_apic_cmd1() & AV_PENDING) 2027 apic_ret(); 2028 2029 /* generate interrupt at vector on itself only */ 2030 apicadr[APIC_INT_CMD1] = AV_SH_SELF | vector; 2031 2032 intr_restore(flag); 2033 } 2034 2035 /* 2036 * generates an interprocessor interrupt to another CPU 2037 */ 2038 static void 2039 apic_send_ipi(int cpun, int ipl) 2040 { 2041 int vector; 2042 uint_t flag; 2043 2044 vector = apic_resv_vector[ipl]; 2045 2046 flag = intr_clear(); 2047 2048 while (get_apic_cmd1() & AV_PENDING) 2049 apic_ret(); 2050 2051 apicadr[APIC_INT_CMD2] = 2052 apic_cpus[cpun].aci_local_id << APIC_ICR_ID_BIT_OFFSET; 2053 apicadr[APIC_INT_CMD1] = vector; 2054 2055 intr_restore(flag); 2056 } 2057 2058 2059 /*ARGSUSED*/ 2060 static void 2061 apic_set_idlecpu(processorid_t cpun) 2062 { 2063 } 2064 2065 /*ARGSUSED*/ 2066 static void 2067 apic_unset_idlecpu(processorid_t cpun) 2068 { 2069 } 2070 2071 2072 static void 2073 apic_ret() 2074 { 2075 } 2076 2077 static int 2078 get_apic_cmd1() 2079 { 2080 return (apicadr[APIC_INT_CMD1]); 2081 } 2082 2083 static int 2084 get_apic_pri() 2085 { 2086 #if defined(__amd64) 2087 return ((int)getcr8()); 2088 #else 2089 return (apicadr[APIC_TASK_REG]); 2090 #endif 2091 } 2092 2093 /* 2094 * If apic_coarse_time == 1, then apic_gettime() is used instead of 2095 * apic_gethrtime(). This is used for performance instead of accuracy. 2096 */ 2097 2098 static hrtime_t 2099 apic_gettime() 2100 { 2101 int old_hrtime_stamp; 2102 hrtime_t temp; 2103 2104 /* 2105 * In one-shot mode, we do not keep time, so if anyone 2106 * calls psm_gettime() directly, we vector over to 2107 * gethrtime(). 2108 * one-shot mode MUST NOT be enabled if this psm is the source of 2109 * hrtime. 2110 */ 2111 2112 if (apic_oneshot) 2113 return (gethrtime()); 2114 2115 2116 gettime_again: 2117 while ((old_hrtime_stamp = apic_hrtime_stamp) & 1) 2118 apic_ret(); 2119 2120 temp = apic_nsec_since_boot; 2121 2122 if (apic_hrtime_stamp != old_hrtime_stamp) { /* got an interrupt */ 2123 goto gettime_again; 2124 } 2125 return (temp); 2126 } 2127 2128 /* 2129 * Here we return the number of nanoseconds since booting. Note every 2130 * clock interrupt increments apic_nsec_since_boot by the appropriate 2131 * amount. 2132 */ 2133 static hrtime_t 2134 apic_gethrtime() 2135 { 2136 int curr_timeval, countval, elapsed_ticks, oflags; 2137 int old_hrtime_stamp, status; 2138 hrtime_t temp; 2139 uchar_t cpun; 2140 2141 2142 /* 2143 * In one-shot mode, we do not keep time, so if anyone 2144 * calls psm_gethrtime() directly, we vector over to 2145 * gethrtime(). 2146 * one-shot mode MUST NOT be enabled if this psm is the source of 2147 * hrtime. 2148 */ 2149 2150 if (apic_oneshot) 2151 return (gethrtime()); 2152 2153 oflags = intr_clear(); /* prevent migration */ 2154 2155 cpun = (uchar_t)((uint_t)apicadr[APIC_LID_REG] >> APIC_ID_BIT_OFFSET); 2156 2157 lock_set(&apic_gethrtime_lock); 2158 2159 gethrtime_again: 2160 while ((old_hrtime_stamp = apic_hrtime_stamp) & 1) 2161 apic_ret(); 2162 2163 /* 2164 * Check to see which CPU we are on. Note the time is kept on 2165 * the local APIC of CPU 0. If on CPU 0, simply read the current 2166 * counter. If on another CPU, issue a remote read command to CPU 0. 2167 */ 2168 if (cpun == apic_cpus[0].aci_local_id) { 2169 countval = apicadr[APIC_CURR_COUNT]; 2170 } else { 2171 while (get_apic_cmd1() & AV_PENDING) 2172 apic_ret(); 2173 2174 apicadr[APIC_INT_CMD2] = 2175 apic_cpus[0].aci_local_id << APIC_ICR_ID_BIT_OFFSET; 2176 apicadr[APIC_INT_CMD1] = APIC_CURR_ADD|AV_REMOTE; 2177 2178 while ((status = get_apic_cmd1()) & AV_READ_PENDING) 2179 apic_ret(); 2180 2181 if (status & AV_REMOTE_STATUS) /* 1 = valid */ 2182 countval = apicadr[APIC_REMOTE_READ]; 2183 else { /* 0 = invalid */ 2184 apic_remote_hrterr++; 2185 /* 2186 * return last hrtime right now, will need more 2187 * testing if change to retry 2188 */ 2189 temp = apic_last_hrtime; 2190 2191 lock_clear(&apic_gethrtime_lock); 2192 2193 intr_restore(oflags); 2194 2195 return (temp); 2196 } 2197 } 2198 if (countval > last_count_read) 2199 countval = 0; 2200 else 2201 last_count_read = countval; 2202 2203 elapsed_ticks = apic_hertz_count - countval; 2204 2205 curr_timeval = elapsed_ticks * apic_nsec_per_tick; 2206 temp = apic_nsec_since_boot + curr_timeval; 2207 2208 if (apic_hrtime_stamp != old_hrtime_stamp) { /* got an interrupt */ 2209 /* we might have clobbered last_count_read. Restore it */ 2210 last_count_read = apic_hertz_count; 2211 goto gethrtime_again; 2212 } 2213 2214 if (temp < apic_last_hrtime) { 2215 /* return last hrtime if error occurs */ 2216 apic_hrtime_error++; 2217 temp = apic_last_hrtime; 2218 } 2219 else 2220 apic_last_hrtime = temp; 2221 2222 lock_clear(&apic_gethrtime_lock); 2223 intr_restore(oflags); 2224 2225 return (temp); 2226 } 2227 2228 /* apic NMI handler */ 2229 /*ARGSUSED*/ 2230 static void 2231 apic_nmi_intr(caddr_t arg) 2232 { 2233 if (apic_shutdown_processors) { 2234 apic_disable_local_apic(); 2235 return; 2236 } 2237 2238 if (lock_try(&apic_nmi_lock)) { 2239 if (apic_kmdb_on_nmi) { 2240 if (psm_debugger() == 0) { 2241 cmn_err(CE_PANIC, 2242 "NMI detected, kmdb is not available."); 2243 } else { 2244 debug_enter("\nNMI detected, entering kmdb.\n"); 2245 } 2246 } else { 2247 if (apic_panic_on_nmi) { 2248 /* Keep panic from entering kmdb. */ 2249 nopanicdebug = 1; 2250 cmn_err(CE_PANIC, "pcplusmp: NMI received"); 2251 } else { 2252 /* 2253 * prom_printf is the best shot we have 2254 * of something which is problem free from 2255 * high level/NMI type of interrupts 2256 */ 2257 prom_printf("pcplusmp: NMI received\n"); 2258 apic_error |= APIC_ERR_NMI; 2259 apic_num_nmis++; 2260 } 2261 } 2262 lock_clear(&apic_nmi_lock); 2263 } 2264 } 2265 2266 /* 2267 * Add mask bits to disable interrupt vector from happening 2268 * at or above IPL. In addition, it should remove mask bits 2269 * to enable interrupt vectors below the given IPL. 2270 * 2271 * Both add and delspl are complicated by the fact that different interrupts 2272 * may share IRQs. This can happen in two ways. 2273 * 1. The same H/W line is shared by more than 1 device 2274 * 1a. with interrupts at different IPLs 2275 * 1b. with interrupts at same IPL 2276 * 2. We ran out of vectors at a given IPL and started sharing vectors. 2277 * 1b and 2 should be handled gracefully, except for the fact some ISRs 2278 * will get called often when no interrupt is pending for the device. 2279 * For 1a, we just hope that the machine blows up with the person who 2280 * set it up that way!. In the meantime, we handle it at the higher IPL. 2281 */ 2282 /*ARGSUSED*/ 2283 static int 2284 apic_addspl(int irqno, int ipl, int min_ipl, int max_ipl) 2285 { 2286 uchar_t vector; 2287 int iflag; 2288 apic_irq_t *irqptr, *irqheadptr; 2289 int irqindex; 2290 2291 ASSERT(max_ipl <= UCHAR_MAX); 2292 irqindex = IRQINDEX(irqno); 2293 2294 if ((irqindex == -1) || (!apic_irq_table[irqindex])) 2295 return (PSM_FAILURE); 2296 2297 irqptr = irqheadptr = apic_irq_table[irqindex]; 2298 2299 DDI_INTR_IMPLDBG((CE_CONT, "apic_addspl: dip=0x%p type=%d irqno=0x%x " 2300 "vector=0x%x\n", (void *)irqptr->airq_dip, 2301 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector)); 2302 2303 while (irqptr) { 2304 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno) 2305 break; 2306 irqptr = irqptr->airq_next; 2307 } 2308 irqptr->airq_share++; 2309 2310 /* return if it is not hardware interrupt */ 2311 if (irqptr->airq_mps_intr_index == RESERVE_INDEX) 2312 return (PSM_SUCCESS); 2313 2314 /* Or if there are more interupts at a higher IPL */ 2315 if (ipl != max_ipl) 2316 return (PSM_SUCCESS); 2317 2318 /* 2319 * if apic_picinit() has not been called yet, just return. 2320 * At the end of apic_picinit(), we will call setup_io_intr(). 2321 */ 2322 2323 if (!apic_flag) 2324 return (PSM_SUCCESS); 2325 2326 iflag = intr_clear(); 2327 2328 /* 2329 * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate, 2330 * return failure. Not very elegant, but then we hope the 2331 * machine will blow up with ... 2332 */ 2333 if (irqptr->airq_ipl != max_ipl) { 2334 vector = apic_allocate_vector(max_ipl, irqindex, 1); 2335 if (vector == 0) { 2336 intr_restore(iflag); 2337 irqptr->airq_share--; 2338 return (PSM_FAILURE); 2339 } 2340 irqptr = irqheadptr; 2341 apic_mark_vector(irqptr->airq_vector, vector); 2342 while (irqptr) { 2343 irqptr->airq_vector = vector; 2344 irqptr->airq_ipl = (uchar_t)max_ipl; 2345 /* 2346 * reprogram irq being added and every one else 2347 * who is not in the UNINIT state 2348 */ 2349 if ((VIRTIRQ(irqindex, irqptr->airq_share_id) == 2350 irqno) || (irqptr->airq_temp_cpu != IRQ_UNINIT)) { 2351 apic_record_rdt_entry(irqptr, irqindex); 2352 (void) apic_setup_io_intr(irqptr, irqindex); 2353 } 2354 irqptr = irqptr->airq_next; 2355 } 2356 intr_restore(iflag); 2357 return (PSM_SUCCESS); 2358 } 2359 2360 ASSERT(irqptr); 2361 (void) apic_setup_io_intr(irqptr, irqindex); 2362 intr_restore(iflag); 2363 return (PSM_SUCCESS); 2364 } 2365 2366 /* 2367 * Recompute mask bits for the given interrupt vector. 2368 * If there is no interrupt servicing routine for this 2369 * vector, this function should disable interrupt vector 2370 * from happening at all IPLs. If there are still 2371 * handlers using the given vector, this function should 2372 * disable the given vector from happening below the lowest 2373 * IPL of the remaining hadlers. 2374 */ 2375 /*ARGSUSED*/ 2376 static int 2377 apic_delspl(int irqno, int ipl, int min_ipl, int max_ipl) 2378 { 2379 uchar_t vector, bind_cpu; 2380 int iflag, intin, irqindex; 2381 volatile int32_t *ioapic; 2382 apic_irq_t *irqptr, *irqheadptr; 2383 2384 irqindex = IRQINDEX(irqno); 2385 irqptr = irqheadptr = apic_irq_table[irqindex]; 2386 2387 DDI_INTR_IMPLDBG((CE_CONT, "apic_delspl: dip=0x%p type=%d irqno=0x%x " 2388 "vector=0x%x\n", (void *)irqptr->airq_dip, 2389 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector)); 2390 2391 while (irqptr) { 2392 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno) 2393 break; 2394 irqptr = irqptr->airq_next; 2395 } 2396 ASSERT(irqptr); 2397 2398 irqptr->airq_share--; 2399 2400 if (ipl < max_ipl) 2401 return (PSM_SUCCESS); 2402 2403 /* return if it is not hardware interrupt */ 2404 if (irqptr->airq_mps_intr_index == RESERVE_INDEX) 2405 return (PSM_SUCCESS); 2406 2407 if (!apic_flag) { 2408 /* 2409 * Clear irq_struct. If two devices shared an intpt 2410 * line & 1 unloaded before picinit, we are hosed. But, then 2411 * we hope the machine will ... 2412 */ 2413 irqptr->airq_mps_intr_index = FREE_INDEX; 2414 irqptr->airq_temp_cpu = IRQ_UNINIT; 2415 apic_free_vector(irqptr->airq_vector); 2416 return (PSM_SUCCESS); 2417 } 2418 /* 2419 * Downgrade vector to new max_ipl if needed.If we cannot allocate, 2420 * use old IPL. Not very elegant, but then we hope ... 2421 */ 2422 if ((irqptr->airq_ipl != max_ipl) && (max_ipl != PSM_INVALID_IPL)) { 2423 apic_irq_t *irqp; 2424 if (vector = apic_allocate_vector(max_ipl, irqno, 1)) { 2425 apic_mark_vector(irqheadptr->airq_vector, vector); 2426 irqp = irqheadptr; 2427 while (irqp) { 2428 irqp->airq_vector = vector; 2429 irqp->airq_ipl = (uchar_t)max_ipl; 2430 if (irqp->airq_temp_cpu != IRQ_UNINIT) { 2431 apic_record_rdt_entry(irqp, irqindex); 2432 (void) apic_setup_io_intr(irqp, 2433 irqindex); 2434 } 2435 irqp = irqp->airq_next; 2436 } 2437 } 2438 } 2439 2440 if (irqptr->airq_share) 2441 return (PSM_SUCCESS); 2442 2443 ioapic = apicioadr[irqptr->airq_ioapicindex]; 2444 intin = irqptr->airq_intin_no; 2445 iflag = intr_clear(); 2446 lock_set(&apic_ioapic_lock); 2447 ioapic[APIC_IO_REG] = APIC_RDT_CMD + 2 * intin; 2448 ioapic[APIC_IO_DATA] = AV_MASK; 2449 2450 /* Disable the MSI/X vector */ 2451 if (APIC_IS_MSI_OR_MSIX_INDEX(irqptr->airq_mps_intr_index)) { 2452 int type = (irqptr->airq_mps_intr_index == MSI_INDEX) ? 2453 DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX; 2454 2455 /* 2456 * Make sure we only disable on the last 2457 * of the multi-MSI support 2458 */ 2459 if (i_ddi_intr_get_current_nintrs(irqptr->airq_dip) == 1) { 2460 (void) pci_msi_unconfigure(irqptr->airq_dip, type, 2461 irqptr->airq_ioapicindex); 2462 2463 (void) pci_msi_disable_mode(irqptr->airq_dip, type, 2464 irqptr->airq_ioapicindex); 2465 } 2466 } 2467 2468 if (max_ipl == PSM_INVALID_IPL) { 2469 ASSERT(irqheadptr == irqptr); 2470 bind_cpu = irqptr->airq_temp_cpu; 2471 if (((uchar_t)bind_cpu != IRQ_UNBOUND) && 2472 ((uchar_t)bind_cpu != IRQ_UNINIT)) { 2473 ASSERT((bind_cpu & ~IRQ_USER_BOUND) < apic_nproc); 2474 if (bind_cpu & IRQ_USER_BOUND) { 2475 /* If hardbound, temp_cpu == cpu */ 2476 bind_cpu &= ~IRQ_USER_BOUND; 2477 apic_cpus[bind_cpu].aci_bound--; 2478 } else 2479 apic_cpus[bind_cpu].aci_temp_bound--; 2480 } 2481 lock_clear(&apic_ioapic_lock); 2482 intr_restore(iflag); 2483 irqptr->airq_temp_cpu = IRQ_UNINIT; 2484 irqptr->airq_mps_intr_index = FREE_INDEX; 2485 apic_free_vector(irqptr->airq_vector); 2486 return (PSM_SUCCESS); 2487 } 2488 lock_clear(&apic_ioapic_lock); 2489 intr_restore(iflag); 2490 2491 mutex_enter(&airq_mutex); 2492 if ((irqptr == apic_irq_table[irqindex])) { 2493 apic_irq_t *oldirqptr; 2494 /* Move valid irq entry to the head */ 2495 irqheadptr = oldirqptr = irqptr; 2496 irqptr = irqptr->airq_next; 2497 ASSERT(irqptr); 2498 while (irqptr) { 2499 if (irqptr->airq_mps_intr_index != FREE_INDEX) 2500 break; 2501 oldirqptr = irqptr; 2502 irqptr = irqptr->airq_next; 2503 } 2504 /* remove all invalid ones from the beginning */ 2505 apic_irq_table[irqindex] = irqptr; 2506 /* 2507 * and link them back after the head. The invalid ones 2508 * begin with irqheadptr and end at oldirqptr 2509 */ 2510 oldirqptr->airq_next = irqptr->airq_next; 2511 irqptr->airq_next = irqheadptr; 2512 } 2513 mutex_exit(&airq_mutex); 2514 2515 irqptr->airq_temp_cpu = IRQ_UNINIT; 2516 irqptr->airq_mps_intr_index = FREE_INDEX; 2517 return (PSM_SUCCESS); 2518 } 2519 2520 /* 2521 * Return HW interrupt number corresponding to the given IPL 2522 */ 2523 /*ARGSUSED*/ 2524 static int 2525 apic_softlvl_to_irq(int ipl) 2526 { 2527 /* 2528 * Do not use apic to trigger soft interrupt. 2529 * It will cause the system to hang when 2 hardware interrupts 2530 * at the same priority with the softint are already accepted 2531 * by the apic. Cause the AV_PENDING bit will not be cleared 2532 * until one of the hardware interrupt is eoi'ed. If we need 2533 * to send an ipi at this time, we will end up looping forever 2534 * to wait for the AV_PENDING bit to clear. 2535 */ 2536 return (PSM_SV_SOFTWARE); 2537 } 2538 2539 static int 2540 apic_post_cpu_start() 2541 { 2542 int i, cpun; 2543 apic_irq_t *irq_ptr; 2544 2545 apic_init_intr(); 2546 2547 /* 2548 * since some systems don't enable the internal cache on the non-boot 2549 * cpus, so we have to enable them here 2550 */ 2551 setcr0(getcr0() & ~(0x60000000)); 2552 2553 while (get_apic_cmd1() & AV_PENDING) 2554 apic_ret(); 2555 2556 cpun = psm_get_cpu_id(); 2557 apic_cpus[cpun].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE; 2558 2559 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) { 2560 irq_ptr = apic_irq_table[i]; 2561 if ((irq_ptr == NULL) || 2562 ((irq_ptr->airq_cpu & ~IRQ_USER_BOUND) != cpun)) 2563 continue; 2564 2565 while (irq_ptr) { 2566 if (irq_ptr->airq_temp_cpu != IRQ_UNINIT) 2567 (void) apic_rebind(irq_ptr, cpun, 1, IMMEDIATE); 2568 irq_ptr = irq_ptr->airq_next; 2569 } 2570 } 2571 2572 return (PSM_SUCCESS); 2573 } 2574 2575 processorid_t 2576 apic_get_next_processorid(processorid_t cpu_id) 2577 { 2578 2579 int i; 2580 2581 if (cpu_id == -1) 2582 return ((processorid_t)0); 2583 2584 for (i = cpu_id + 1; i < NCPU; i++) { 2585 if (apic_cpumask & (1 << i)) 2586 return (i); 2587 } 2588 2589 return ((processorid_t)-1); 2590 } 2591 2592 2593 /* 2594 * type == -1 indicates it is an internal request. Do not change 2595 * resv_vector for these requests 2596 */ 2597 static int 2598 apic_get_ipivect(int ipl, int type) 2599 { 2600 uchar_t vector; 2601 int irq; 2602 2603 if (irq = apic_allocate_irq(APIC_VECTOR(ipl))) { 2604 if (vector = apic_allocate_vector(ipl, irq, 1)) { 2605 apic_irq_table[irq]->airq_mps_intr_index = 2606 RESERVE_INDEX; 2607 apic_irq_table[irq]->airq_vector = vector; 2608 if (type != -1) { 2609 apic_resv_vector[ipl] = vector; 2610 } 2611 return (irq); 2612 } 2613 } 2614 apic_error |= APIC_ERR_GET_IPIVECT_FAIL; 2615 return (-1); /* shouldn't happen */ 2616 } 2617 2618 static int 2619 apic_getclkirq(int ipl) 2620 { 2621 int irq; 2622 2623 if ((irq = apic_get_ipivect(ipl, -1)) == -1) 2624 return (-1); 2625 /* 2626 * Note the vector in apic_clkvect for per clock handling. 2627 */ 2628 apic_clkvect = apic_irq_table[irq]->airq_vector - APIC_BASE_VECT; 2629 APIC_VERBOSE_IOAPIC((CE_NOTE, "get_clkirq: vector = %x\n", 2630 apic_clkvect)); 2631 return (irq); 2632 } 2633 2634 /* 2635 * Return the number of APIC clock ticks elapsed for 8245 to decrement 2636 * (APIC_TIME_COUNT + pit_ticks_adj) ticks. 2637 */ 2638 static uint_t 2639 apic_calibrate(volatile uint32_t *addr, uint16_t *pit_ticks_adj) 2640 { 2641 uint8_t pit_tick_lo; 2642 uint16_t pit_tick, target_pit_tick; 2643 uint32_t start_apic_tick, end_apic_tick; 2644 int iflag; 2645 2646 addr += APIC_CURR_COUNT; 2647 2648 iflag = intr_clear(); 2649 2650 do { 2651 pit_tick_lo = inb(PITCTR0_PORT); 2652 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo; 2653 } while (pit_tick < APIC_TIME_MIN || 2654 pit_tick_lo <= APIC_LB_MIN || pit_tick_lo >= APIC_LB_MAX); 2655 2656 /* 2657 * Wait for the 8254 to decrement by 5 ticks to ensure 2658 * we didn't start in the middle of a tick. 2659 * Compare with 0x10 for the wrap around case. 2660 */ 2661 target_pit_tick = pit_tick - 5; 2662 do { 2663 pit_tick_lo = inb(PITCTR0_PORT); 2664 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo; 2665 } while (pit_tick > target_pit_tick || pit_tick_lo < 0x10); 2666 2667 start_apic_tick = *addr; 2668 2669 /* 2670 * Wait for the 8254 to decrement by 2671 * (APIC_TIME_COUNT + pit_ticks_adj) ticks 2672 */ 2673 target_pit_tick = pit_tick - APIC_TIME_COUNT; 2674 do { 2675 pit_tick_lo = inb(PITCTR0_PORT); 2676 pit_tick = (inb(PITCTR0_PORT) << 8) | pit_tick_lo; 2677 } while (pit_tick > target_pit_tick || pit_tick_lo < 0x10); 2678 2679 end_apic_tick = *addr; 2680 2681 *pit_ticks_adj = target_pit_tick - pit_tick; 2682 2683 intr_restore(iflag); 2684 2685 return (start_apic_tick - end_apic_tick); 2686 } 2687 2688 /* 2689 * Initialise the APIC timer on the local APIC of CPU 0 to the desired 2690 * frequency. Note at this stage in the boot sequence, the boot processor 2691 * is the only active processor. 2692 * hertz value of 0 indicates a one-shot mode request. In this case 2693 * the function returns the resolution (in nanoseconds) for the hardware 2694 * timer interrupt. If one-shot mode capability is not available, 2695 * the return value will be 0. apic_enable_oneshot is a global switch 2696 * for disabling the functionality. 2697 * A non-zero positive value for hertz indicates a periodic mode request. 2698 * In this case the hardware will be programmed to generate clock interrupts 2699 * at hertz frequency and returns the resolution of interrupts in 2700 * nanosecond. 2701 */ 2702 2703 static int 2704 apic_clkinit(int hertz) 2705 { 2706 2707 uint_t apic_ticks = 0; 2708 uint_t pit_time; 2709 int ret; 2710 uint16_t pit_ticks_adj; 2711 static int firsttime = 1; 2712 2713 if (firsttime) { 2714 /* first time calibrate */ 2715 2716 apicadr[APIC_DIVIDE_REG] = 0x0; 2717 apicadr[APIC_INIT_COUNT] = APIC_MAXVAL; 2718 2719 /* set periodic interrupt based on CLKIN */ 2720 apicadr[APIC_LOCAL_TIMER] = 2721 (apic_clkvect + APIC_BASE_VECT) | AV_TIME; 2722 tenmicrosec(); 2723 2724 apic_ticks = apic_calibrate(apicadr, &pit_ticks_adj); 2725 2726 apicadr[APIC_LOCAL_TIMER] = 2727 (apic_clkvect + APIC_BASE_VECT) | AV_MASK; 2728 /* 2729 * pit time is the amount of real time (in nanoseconds ) it took 2730 * the 8254 to decrement (APIC_TIME_COUNT + pit_ticks_adj) ticks 2731 */ 2732 pit_time = ((longlong_t)(APIC_TIME_COUNT + 2733 pit_ticks_adj) * NANOSEC) / PIT_HZ; 2734 2735 /* 2736 * Determine the number of nanoseconds per APIC clock tick 2737 * and then determine how many APIC ticks to interrupt at the 2738 * desired frequency 2739 */ 2740 apic_nsec_per_tick = pit_time / apic_ticks; 2741 if (apic_nsec_per_tick == 0) 2742 apic_nsec_per_tick = 1; 2743 2744 /* the interval timer initial count is 32 bit max */ 2745 apic_nsec_max = (hrtime_t)apic_nsec_per_tick * APIC_MAXVAL; 2746 firsttime = 0; 2747 } 2748 2749 if (hertz != 0) { 2750 /* periodic */ 2751 apic_nsec_per_intr = NANOSEC / hertz; 2752 apic_hertz_count = (longlong_t)apic_nsec_per_intr / 2753 apic_nsec_per_tick; 2754 apic_sample_factor_redistribution = hertz + 1; 2755 } 2756 2757 apic_int_busy_mark = (apic_int_busy_mark * 2758 apic_sample_factor_redistribution) / 100; 2759 apic_int_free_mark = (apic_int_free_mark * 2760 apic_sample_factor_redistribution) / 100; 2761 apic_diff_for_redistribution = (apic_diff_for_redistribution * 2762 apic_sample_factor_redistribution) / 100; 2763 2764 if (hertz == 0) { 2765 /* requested one_shot */ 2766 if (!apic_oneshot_enable) 2767 return (0); 2768 apic_oneshot = 1; 2769 ret = (int)apic_nsec_per_tick; 2770 } else { 2771 /* program the local APIC to interrupt at the given frequency */ 2772 apicadr[APIC_INIT_COUNT] = apic_hertz_count; 2773 apicadr[APIC_LOCAL_TIMER] = 2774 (apic_clkvect + APIC_BASE_VECT) | AV_TIME; 2775 apic_oneshot = 0; 2776 ret = NANOSEC / hertz; 2777 } 2778 2779 return (ret); 2780 2781 } 2782 2783 /* 2784 * apic_preshutdown: 2785 * Called early in shutdown whilst we can still access filesystems to do 2786 * things like loading modules which will be required to complete shutdown 2787 * after filesystems are all unmounted. 2788 */ 2789 static void 2790 apic_preshutdown(int cmd, int fcn) 2791 { 2792 APIC_VERBOSE_POWEROFF(("apic_preshutdown(%d,%d); m=%d a=%d\n", 2793 cmd, fcn, apic_poweroff_method, apic_enable_acpi)); 2794 2795 if ((cmd != A_SHUTDOWN) || (fcn != AD_POWEROFF)) { 2796 return; 2797 } 2798 } 2799 2800 static void 2801 apic_shutdown(int cmd, int fcn) 2802 { 2803 int iflag, restarts, attempts; 2804 int i, j; 2805 volatile int32_t *ioapic; 2806 uchar_t byte; 2807 2808 /* Send NMI to all CPUs except self to do per processor shutdown */ 2809 iflag = intr_clear(); 2810 while (get_apic_cmd1() & AV_PENDING) 2811 apic_ret(); 2812 apic_shutdown_processors = 1; 2813 apicadr[APIC_INT_CMD1] = AV_NMI | AV_LEVEL | AV_SH_ALL_EXCSELF; 2814 2815 /* restore cmos shutdown byte before reboot */ 2816 if (apic_cmos_ssb_set) { 2817 outb(CMOS_ADDR, SSB); 2818 outb(CMOS_DATA, 0); 2819 } 2820 /* Disable the I/O APIC redirection entries */ 2821 for (j = 0; j < apic_io_max; j++) { 2822 int intin_max; 2823 ioapic = apicioadr[j]; 2824 ioapic[APIC_IO_REG] = APIC_VERS_CMD; 2825 /* Bits 23-16 define the maximum redirection entries */ 2826 intin_max = (ioapic[APIC_IO_DATA] >> 16) & 0xff; 2827 for (i = 0; i < intin_max; i++) { 2828 ioapic[APIC_IO_REG] = APIC_RDT_CMD + 2 * i; 2829 ioapic[APIC_IO_DATA] = AV_MASK; 2830 } 2831 } 2832 2833 /* disable apic mode if imcr present */ 2834 if (apic_imcrp) { 2835 outb(APIC_IMCR_P1, (uchar_t)APIC_IMCR_SELECT); 2836 outb(APIC_IMCR_P2, (uchar_t)APIC_IMCR_PIC); 2837 } 2838 2839 apic_disable_local_apic(); 2840 2841 intr_restore(iflag); 2842 2843 if ((cmd != A_SHUTDOWN) || (fcn != AD_POWEROFF)) { 2844 return; 2845 } 2846 2847 switch (apic_poweroff_method) { 2848 case APIC_POWEROFF_VIA_RTC: 2849 2850 /* select the extended NVRAM bank in the RTC */ 2851 outb(CMOS_ADDR, RTC_REGA); 2852 byte = inb(CMOS_DATA); 2853 outb(CMOS_DATA, (byte | EXT_BANK)); 2854 2855 outb(CMOS_ADDR, PFR_REG); 2856 2857 /* for Predator must toggle the PAB bit */ 2858 byte = inb(CMOS_DATA); 2859 2860 /* 2861 * clear power active bar, wakeup alarm and 2862 * kickstart 2863 */ 2864 byte &= ~(PAB_CBIT | WF_FLAG | KS_FLAG); 2865 outb(CMOS_DATA, byte); 2866 2867 /* delay before next write */ 2868 drv_usecwait(1000); 2869 2870 /* for S40 the following would suffice */ 2871 byte = inb(CMOS_DATA); 2872 2873 /* power active bar control bit */ 2874 byte |= PAB_CBIT; 2875 outb(CMOS_DATA, byte); 2876 2877 break; 2878 2879 case APIC_POWEROFF_VIA_ASPEN_BMC: 2880 restarts = 0; 2881 restart_aspen_bmc: 2882 if (++restarts == 3) 2883 break; 2884 attempts = 0; 2885 do { 2886 byte = inb(MISMIC_FLAG_REGISTER); 2887 byte &= MISMIC_BUSY_MASK; 2888 if (byte != 0) { 2889 drv_usecwait(1000); 2890 if (attempts >= 3) 2891 goto restart_aspen_bmc; 2892 ++attempts; 2893 } 2894 } while (byte != 0); 2895 outb(MISMIC_CNTL_REGISTER, CC_SMS_GET_STATUS); 2896 byte = inb(MISMIC_FLAG_REGISTER); 2897 byte |= 0x1; 2898 outb(MISMIC_FLAG_REGISTER, byte); 2899 i = 0; 2900 for (; i < (sizeof (aspen_bmc)/sizeof (aspen_bmc[0])); 2901 i++) { 2902 attempts = 0; 2903 do { 2904 byte = inb(MISMIC_FLAG_REGISTER); 2905 byte &= MISMIC_BUSY_MASK; 2906 if (byte != 0) { 2907 drv_usecwait(1000); 2908 if (attempts >= 3) 2909 goto restart_aspen_bmc; 2910 ++attempts; 2911 } 2912 } while (byte != 0); 2913 outb(MISMIC_CNTL_REGISTER, aspen_bmc[i].cntl); 2914 outb(MISMIC_DATA_REGISTER, aspen_bmc[i].data); 2915 byte = inb(MISMIC_FLAG_REGISTER); 2916 byte |= 0x1; 2917 outb(MISMIC_FLAG_REGISTER, byte); 2918 } 2919 break; 2920 2921 case APIC_POWEROFF_VIA_SITKA_BMC: 2922 restarts = 0; 2923 restart_sitka_bmc: 2924 if (++restarts == 3) 2925 break; 2926 attempts = 0; 2927 do { 2928 byte = inb(SMS_STATUS_REGISTER); 2929 byte &= SMS_STATE_MASK; 2930 if ((byte == SMS_READ_STATE) || 2931 (byte == SMS_WRITE_STATE)) { 2932 drv_usecwait(1000); 2933 if (attempts >= 3) 2934 goto restart_sitka_bmc; 2935 ++attempts; 2936 } 2937 } while ((byte == SMS_READ_STATE) || 2938 (byte == SMS_WRITE_STATE)); 2939 outb(SMS_COMMAND_REGISTER, SMS_GET_STATUS); 2940 i = 0; 2941 for (; i < (sizeof (sitka_bmc)/sizeof (sitka_bmc[0])); 2942 i++) { 2943 attempts = 0; 2944 do { 2945 byte = inb(SMS_STATUS_REGISTER); 2946 byte &= SMS_IBF_MASK; 2947 if (byte != 0) { 2948 drv_usecwait(1000); 2949 if (attempts >= 3) 2950 goto restart_sitka_bmc; 2951 ++attempts; 2952 } 2953 } while (byte != 0); 2954 outb(sitka_bmc[i].port, sitka_bmc[i].data); 2955 } 2956 break; 2957 2958 case APIC_POWEROFF_NONE: 2959 2960 /* If no APIC direct method, we will try using ACPI */ 2961 if (apic_enable_acpi) { 2962 if (acpi_poweroff() == 1) 2963 return; 2964 } else 2965 return; 2966 2967 break; 2968 } 2969 /* 2970 * Wait a limited time here for power to go off. 2971 * If the power does not go off, then there was a 2972 * problem and we should continue to the halt which 2973 * prints a message for the user to press a key to 2974 * reboot. 2975 */ 2976 drv_usecwait(7000000); /* wait seven seconds */ 2977 2978 } 2979 2980 /* 2981 * Try and disable all interrupts. We just assign interrupts to other 2982 * processors based on policy. If any were bound by user request, we 2983 * let them continue and return failure. We do not bother to check 2984 * for cache affinity while rebinding. 2985 */ 2986 2987 static int 2988 apic_disable_intr(processorid_t cpun) 2989 { 2990 int bind_cpu = 0, i, hardbound = 0, iflag; 2991 apic_irq_t *irq_ptr; 2992 2993 if (cpun == 0) 2994 return (PSM_FAILURE); 2995 2996 iflag = intr_clear(); 2997 lock_set(&apic_ioapic_lock); 2998 apic_cpus[cpun].aci_status &= ~APIC_CPU_INTR_ENABLE; 2999 lock_clear(&apic_ioapic_lock); 3000 intr_restore(iflag); 3001 apic_cpus[cpun].aci_curipl = 0; 3002 i = apic_min_device_irq; 3003 for (; i <= apic_max_device_irq; i++) { 3004 /* 3005 * If there are bound interrupts on this cpu, then 3006 * rebind them to other processors. 3007 */ 3008 if ((irq_ptr = apic_irq_table[i]) != NULL) { 3009 ASSERT((irq_ptr->airq_temp_cpu == IRQ_UNBOUND) || 3010 (irq_ptr->airq_temp_cpu == IRQ_UNINIT) || 3011 ((irq_ptr->airq_temp_cpu & ~IRQ_USER_BOUND) < 3012 apic_nproc)); 3013 3014 if (irq_ptr->airq_temp_cpu == (cpun | IRQ_USER_BOUND)) { 3015 hardbound = 1; 3016 continue; 3017 } 3018 3019 if (irq_ptr->airq_temp_cpu == cpun) { 3020 do { 3021 apic_next_bind_cpu += 2; 3022 bind_cpu = apic_next_bind_cpu / 2; 3023 if (bind_cpu >= apic_nproc) { 3024 apic_next_bind_cpu = 1; 3025 bind_cpu = 0; 3026 3027 } 3028 } while (apic_rebind_all(irq_ptr, bind_cpu, 1)); 3029 } 3030 } 3031 } 3032 if (hardbound) { 3033 cmn_err(CE_WARN, "Could not disable interrupts on %d" 3034 "due to user bound interrupts", cpun); 3035 return (PSM_FAILURE); 3036 } 3037 else 3038 return (PSM_SUCCESS); 3039 } 3040 3041 static void 3042 apic_enable_intr(processorid_t cpun) 3043 { 3044 int i, iflag; 3045 apic_irq_t *irq_ptr; 3046 3047 iflag = intr_clear(); 3048 lock_set(&apic_ioapic_lock); 3049 apic_cpus[cpun].aci_status |= APIC_CPU_INTR_ENABLE; 3050 lock_clear(&apic_ioapic_lock); 3051 intr_restore(iflag); 3052 3053 i = apic_min_device_irq; 3054 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) { 3055 if ((irq_ptr = apic_irq_table[i]) != NULL) { 3056 if ((irq_ptr->airq_cpu & ~IRQ_USER_BOUND) == cpun) { 3057 (void) apic_rebind_all(irq_ptr, 3058 irq_ptr->airq_cpu, 1); 3059 } 3060 } 3061 } 3062 } 3063 3064 /* 3065 * apic_introp_xlate() replaces apic_translate_irq() and is 3066 * called only from apic_intr_ops(). With the new ADII framework, 3067 * the priority can no longer be retrived through i_ddi_get_intrspec(). 3068 * It has to be passed in from the caller. 3069 */ 3070 int 3071 apic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type) 3072 { 3073 char dev_type[16]; 3074 int dev_len, pci_irq, newirq, bustype, devid, busid, i; 3075 int irqno = ispec->intrspec_vec; 3076 ddi_acc_handle_t cfg_handle; 3077 uchar_t ipin; 3078 struct apic_io_intr *intrp; 3079 iflag_t intr_flag; 3080 APIC_HEADER *hp; 3081 MADT_INTERRUPT_OVERRIDE *isop; 3082 apic_irq_t *airqp; 3083 3084 DDI_INTR_IMPLDBG((CE_CONT, "apic_introp_xlate: dip=0x%p name=%s " 3085 "type=%d irqno=0x%x\n", (void *)dip, ddi_get_name(dip), type, 3086 irqno)); 3087 3088 if (DDI_INTR_IS_MSI_OR_MSIX(type)) { 3089 if ((airqp = apic_find_irq(dip, ispec, type)) != NULL) 3090 return (apic_vector_to_irq[airqp->airq_vector]); 3091 return (apic_setup_irq_table(dip, irqno, NULL, ispec, 3092 NULL, type)); 3093 } 3094 3095 bustype = 0; 3096 3097 /* check if we have already translated this irq */ 3098 mutex_enter(&airq_mutex); 3099 newirq = apic_min_device_irq; 3100 for (; newirq <= apic_max_device_irq; newirq++) { 3101 airqp = apic_irq_table[newirq]; 3102 while (airqp) { 3103 if ((airqp->airq_dip == dip) && 3104 (airqp->airq_origirq == irqno) && 3105 (airqp->airq_mps_intr_index != FREE_INDEX)) { 3106 3107 mutex_exit(&airq_mutex); 3108 return (VIRTIRQ(newirq, airqp->airq_share_id)); 3109 } 3110 airqp = airqp->airq_next; 3111 } 3112 } 3113 mutex_exit(&airq_mutex); 3114 3115 if (apic_defconf) 3116 goto defconf; 3117 3118 if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi)) 3119 goto nonpci; 3120 3121 dev_len = sizeof (dev_type); 3122 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip), 3123 DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type, 3124 &dev_len) != DDI_PROP_SUCCESS) { 3125 goto nonpci; 3126 } 3127 3128 if (strcmp(dev_type, "pci") == 0) { 3129 /* pci device */ 3130 if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0) 3131 goto nonpci; 3132 if (busid == 0 && apic_pci_bus_total == 1) 3133 busid = (int)apic_single_pci_busid; 3134 3135 if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS) 3136 goto nonpci; 3137 ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA; 3138 pci_config_teardown(&cfg_handle); 3139 if (apic_enable_acpi && !apic_use_acpi_madt_only) { 3140 if (apic_acpi_translate_pci_irq(dip, busid, devid, 3141 ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS) 3142 goto nonpci; 3143 3144 intr_flag.bustype = BUS_PCI; 3145 if ((newirq = apic_setup_irq_table(dip, pci_irq, NULL, 3146 ispec, &intr_flag, type)) == -1) 3147 goto nonpci; 3148 return (newirq); 3149 } else { 3150 pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3); 3151 if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid)) 3152 == NULL) { 3153 if ((pci_irq = apic_handle_pci_pci_bridge(dip, 3154 devid, ipin, &intrp)) == -1) 3155 goto nonpci; 3156 } 3157 if ((newirq = apic_setup_irq_table(dip, pci_irq, intrp, 3158 ispec, NULL, type)) == -1) 3159 goto nonpci; 3160 return (newirq); 3161 } 3162 } else if (strcmp(dev_type, "isa") == 0) 3163 bustype = BUS_ISA; 3164 else if (strcmp(dev_type, "eisa") == 0) 3165 bustype = BUS_EISA; 3166 3167 nonpci: 3168 if (apic_enable_acpi && !apic_use_acpi_madt_only) { 3169 /* search iso entries first */ 3170 if (acpi_iso_cnt != 0) { 3171 hp = (APIC_HEADER *)acpi_isop; 3172 i = 0; 3173 while (i < acpi_iso_cnt) { 3174 if (hp->Type == APIC_XRUPT_OVERRIDE) { 3175 isop = (MADT_INTERRUPT_OVERRIDE *)hp; 3176 if (isop->Bus == 0 && 3177 isop->Source == irqno) { 3178 newirq = isop->Interrupt; 3179 intr_flag.intr_po = 3180 isop->Polarity; 3181 intr_flag.intr_el = 3182 isop->TriggerMode; 3183 intr_flag.bustype = BUS_ISA; 3184 3185 return (apic_setup_irq_table( 3186 dip, newirq, NULL, ispec, 3187 &intr_flag, type)); 3188 3189 } 3190 i++; 3191 } 3192 hp = (APIC_HEADER *)(((char *)hp) + 3193 hp->Length); 3194 } 3195 } 3196 intr_flag.intr_po = INTR_PO_ACTIVE_HIGH; 3197 intr_flag.intr_el = INTR_EL_EDGE; 3198 intr_flag.bustype = BUS_ISA; 3199 return (apic_setup_irq_table(dip, irqno, NULL, ispec, 3200 &intr_flag, type)); 3201 } else { 3202 if (bustype == 0) 3203 bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA; 3204 for (i = 0; i < 2; i++) { 3205 if (((busid = apic_find_bus_id(bustype)) != -1) && 3206 ((intrp = apic_find_io_intr_w_busid(irqno, busid)) 3207 != NULL)) { 3208 if ((newirq = apic_setup_irq_table(dip, irqno, 3209 intrp, ispec, NULL, type)) != -1) { 3210 return (newirq); 3211 } 3212 goto defconf; 3213 } 3214 bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA; 3215 } 3216 } 3217 3218 /* MPS default configuration */ 3219 defconf: 3220 newirq = apic_setup_irq_table(dip, irqno, NULL, ispec, NULL, type); 3221 if (newirq == -1) 3222 return (newirq); 3223 ASSERT(IRQINDEX(newirq) == irqno); 3224 ASSERT(apic_irq_table[irqno]); 3225 return (newirq); 3226 } 3227 3228 3229 3230 3231 3232 3233 /* 3234 * On machines with PCI-PCI bridges, a device behind a PCI-PCI bridge 3235 * needs special handling. We may need to chase up the device tree, 3236 * using the PCI-PCI Bridge specification's "rotating IPIN assumptions", 3237 * to find the IPIN at the root bus that relates to the IPIN on the 3238 * subsidiary bus (for ACPI or MP). We may, however, have an entry 3239 * in the MP table or the ACPI namespace for this device itself. 3240 * We handle both cases in the search below. 3241 */ 3242 /* this is the non-acpi version */ 3243 static int 3244 apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno, int child_ipin, 3245 struct apic_io_intr **intrp) 3246 { 3247 dev_info_t *dipp, *dip; 3248 int pci_irq; 3249 ddi_acc_handle_t cfg_handle; 3250 int bridge_devno, bridge_bus; 3251 int ipin; 3252 3253 dip = idip; 3254 3255 /*CONSTCOND*/ 3256 while (1) { 3257 if ((dipp = ddi_get_parent(dip)) == (dev_info_t *)NULL) 3258 return (-1); 3259 if ((pci_config_setup(dipp, &cfg_handle) == DDI_SUCCESS) && 3260 (pci_config_get8(cfg_handle, PCI_CONF_BASCLASS) == 3261 PCI_CLASS_BRIDGE) && (pci_config_get8(cfg_handle, 3262 PCI_CONF_SUBCLASS) == PCI_BRIDGE_PCI)) { 3263 pci_config_teardown(&cfg_handle); 3264 if (acpica_get_bdf(dipp, &bridge_bus, &bridge_devno, 3265 NULL) != 0) 3266 return (-1); 3267 /* 3268 * This is the rotating scheme that Compaq is using 3269 * and documented in the pci to pci spec. Also, if 3270 * the pci to pci bridge is behind another pci to 3271 * pci bridge, then it need to keep transversing 3272 * up until an interrupt entry is found or reach 3273 * the top of the tree 3274 */ 3275 ipin = (child_devno + child_ipin) % PCI_INTD; 3276 if (bridge_bus == 0 && apic_pci_bus_total == 1) 3277 bridge_bus = (int)apic_single_pci_busid; 3278 pci_irq = ((bridge_devno & 0x1f) << 2) | 3279 (ipin & 0x3); 3280 if ((*intrp = apic_find_io_intr_w_busid(pci_irq, 3281 bridge_bus)) != NULL) { 3282 return (pci_irq); 3283 } 3284 dip = dipp; 3285 child_devno = bridge_devno; 3286 child_ipin = ipin; 3287 } else 3288 return (-1); 3289 } 3290 /*LINTED: function will not fall off the bottom */ 3291 } 3292 3293 3294 3295 3296 static uchar_t 3297 acpi_find_ioapic(int irq) 3298 { 3299 int i; 3300 3301 for (i = 0; i < apic_io_max; i++) { 3302 if (irq >= apic_io_vectbase[i] && irq <= apic_io_vectend[i]) 3303 return (i); 3304 } 3305 return (0xFF); /* shouldn't happen */ 3306 } 3307 3308 /* 3309 * See if two irqs are compatible for sharing a vector. 3310 * Currently we only support sharing of PCI devices. 3311 */ 3312 static int 3313 acpi_intr_compatible(iflag_t iflag1, iflag_t iflag2) 3314 { 3315 uint_t level1, po1; 3316 uint_t level2, po2; 3317 3318 /* Assume active high by default */ 3319 po1 = 0; 3320 po2 = 0; 3321 3322 if (iflag1.bustype != iflag2.bustype || iflag1.bustype != BUS_PCI) 3323 return (0); 3324 3325 if (iflag1.intr_el == INTR_EL_CONFORM) 3326 level1 = AV_LEVEL; 3327 else 3328 level1 = (iflag1.intr_el == INTR_EL_LEVEL) ? AV_LEVEL : 0; 3329 3330 if (level1 && ((iflag1.intr_po == INTR_PO_ACTIVE_LOW) || 3331 (iflag1.intr_po == INTR_PO_CONFORM))) 3332 po1 = AV_ACTIVE_LOW; 3333 3334 if (iflag2.intr_el == INTR_EL_CONFORM) 3335 level2 = AV_LEVEL; 3336 else 3337 level2 = (iflag2.intr_el == INTR_EL_LEVEL) ? AV_LEVEL : 0; 3338 3339 if (level2 && ((iflag2.intr_po == INTR_PO_ACTIVE_LOW) || 3340 (iflag2.intr_po == INTR_PO_CONFORM))) 3341 po2 = AV_ACTIVE_LOW; 3342 3343 if ((level1 == level2) && (po1 == po2)) 3344 return (1); 3345 3346 return (0); 3347 } 3348 3349 /* 3350 * Attempt to share vector with someone else 3351 */ 3352 static int 3353 apic_share_vector(int irqno, iflag_t *intr_flagp, short intr_index, int ipl, 3354 uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp) 3355 { 3356 #ifdef DEBUG 3357 apic_irq_t *tmpirqp = NULL; 3358 #endif /* DEBUG */ 3359 apic_irq_t *irqptr, dummyirq; 3360 int newirq, chosen_irq = -1, share = 127; 3361 int lowest, highest, i; 3362 uchar_t share_id; 3363 3364 DDI_INTR_IMPLDBG((CE_CONT, "apic_share_vector: irqno=0x%x " 3365 "intr_index=0x%x ipl=0x%x\n", irqno, intr_index, ipl)); 3366 3367 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK; 3368 lowest = apic_ipltopri[ipl-1] + APIC_VECTOR_PER_IPL; 3369 3370 if (highest < lowest) /* Both ipl and ipl-1 map to same pri */ 3371 lowest -= APIC_VECTOR_PER_IPL; 3372 dummyirq.airq_mps_intr_index = intr_index; 3373 dummyirq.airq_ioapicindex = ioapicindex; 3374 dummyirq.airq_intin_no = ipin; 3375 if (intr_flagp) 3376 dummyirq.airq_iflag = *intr_flagp; 3377 apic_record_rdt_entry(&dummyirq, irqno); 3378 for (i = lowest; i <= highest; i++) { 3379 newirq = apic_vector_to_irq[i]; 3380 if (newirq == APIC_RESV_IRQ) 3381 continue; 3382 irqptr = apic_irq_table[newirq]; 3383 3384 if ((dummyirq.airq_rdt_entry & 0xFF00) != 3385 (irqptr->airq_rdt_entry & 0xFF00)) 3386 /* not compatible */ 3387 continue; 3388 3389 if (irqptr->airq_share < share) { 3390 share = irqptr->airq_share; 3391 chosen_irq = newirq; 3392 } 3393 } 3394 if (chosen_irq != -1) { 3395 /* 3396 * Assign a share id which is free or which is larger 3397 * than the largest one. 3398 */ 3399 share_id = 1; 3400 mutex_enter(&airq_mutex); 3401 irqptr = apic_irq_table[chosen_irq]; 3402 while (irqptr) { 3403 if (irqptr->airq_mps_intr_index == FREE_INDEX) { 3404 share_id = irqptr->airq_share_id; 3405 break; 3406 } 3407 if (share_id <= irqptr->airq_share_id) 3408 share_id = irqptr->airq_share_id + 1; 3409 #ifdef DEBUG 3410 tmpirqp = irqptr; 3411 #endif /* DEBUG */ 3412 irqptr = irqptr->airq_next; 3413 } 3414 if (!irqptr) { 3415 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP); 3416 irqptr->airq_temp_cpu = IRQ_UNINIT; 3417 irqptr->airq_next = 3418 apic_irq_table[chosen_irq]->airq_next; 3419 apic_irq_table[chosen_irq]->airq_next = irqptr; 3420 #ifdef DEBUG 3421 tmpirqp = apic_irq_table[chosen_irq]; 3422 #endif /* DEBUG */ 3423 } 3424 irqptr->airq_mps_intr_index = intr_index; 3425 irqptr->airq_ioapicindex = ioapicindex; 3426 irqptr->airq_intin_no = ipin; 3427 if (intr_flagp) 3428 irqptr->airq_iflag = *intr_flagp; 3429 irqptr->airq_vector = apic_irq_table[chosen_irq]->airq_vector; 3430 irqptr->airq_share_id = share_id; 3431 apic_record_rdt_entry(irqptr, irqno); 3432 *irqptrp = irqptr; 3433 #ifdef DEBUG 3434 /* shuffle the pointers to test apic_delspl path */ 3435 if (tmpirqp) { 3436 tmpirqp->airq_next = irqptr->airq_next; 3437 irqptr->airq_next = apic_irq_table[chosen_irq]; 3438 apic_irq_table[chosen_irq] = irqptr; 3439 } 3440 #endif /* DEBUG */ 3441 mutex_exit(&airq_mutex); 3442 return (VIRTIRQ(chosen_irq, share_id)); 3443 } 3444 return (-1); 3445 } 3446 3447 /* 3448 * 3449 */ 3450 static int 3451 apic_setup_irq_table(dev_info_t *dip, int irqno, struct apic_io_intr *intrp, 3452 struct intrspec *ispec, iflag_t *intr_flagp, int type) 3453 { 3454 int origirq = ispec->intrspec_vec; 3455 uchar_t ipl = ispec->intrspec_pri; 3456 int newirq, intr_index; 3457 uchar_t ipin, ioapic, ioapicindex, vector; 3458 apic_irq_t *irqptr; 3459 major_t major; 3460 dev_info_t *sdip; 3461 3462 DDI_INTR_IMPLDBG((CE_CONT, "apic_setup_irq_table: dip=0x%p type=%d " 3463 "irqno=0x%x origirq=0x%x\n", (void *)dip, type, irqno, origirq)); 3464 3465 ASSERT(ispec != NULL); 3466 3467 major = (dip != NULL) ? ddi_name_to_major(ddi_get_name(dip)) : 0; 3468 3469 if (DDI_INTR_IS_MSI_OR_MSIX(type)) { 3470 /* MSI/X doesn't need to setup ioapic stuffs */ 3471 ioapicindex = 0xff; 3472 ioapic = 0xff; 3473 ipin = (uchar_t)0xff; 3474 intr_index = (type == DDI_INTR_TYPE_MSI) ? MSI_INDEX : 3475 MSIX_INDEX; 3476 mutex_enter(&airq_mutex); 3477 if ((irqno = apic_allocate_irq(APIC_FIRST_FREE_IRQ)) == -1) { 3478 mutex_exit(&airq_mutex); 3479 /* need an irq for MSI/X to index into autovect[] */ 3480 cmn_err(CE_WARN, "No interrupt irq: %s instance %d", 3481 ddi_get_name(dip), ddi_get_instance(dip)); 3482 return (-1); 3483 } 3484 mutex_exit(&airq_mutex); 3485 3486 } else if (intrp != NULL) { 3487 intr_index = (int)(intrp - apic_io_intrp); 3488 ioapic = intrp->intr_destid; 3489 ipin = intrp->intr_destintin; 3490 /* Find ioapicindex. If destid was ALL, we will exit with 0. */ 3491 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--) 3492 if (apic_io_id[ioapicindex] == ioapic) 3493 break; 3494 ASSERT((ioapic == apic_io_id[ioapicindex]) || 3495 (ioapic == INTR_ALL_APIC)); 3496 3497 /* check whether this intin# has been used by another irqno */ 3498 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1) { 3499 return (newirq); 3500 } 3501 3502 } else if (intr_flagp != NULL) { 3503 /* ACPI case */ 3504 intr_index = ACPI_INDEX; 3505 ioapicindex = acpi_find_ioapic(irqno); 3506 ASSERT(ioapicindex != 0xFF); 3507 ioapic = apic_io_id[ioapicindex]; 3508 ipin = irqno - apic_io_vectbase[ioapicindex]; 3509 if (apic_irq_table[irqno] && 3510 apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) { 3511 ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin && 3512 apic_irq_table[irqno]->airq_ioapicindex == 3513 ioapicindex); 3514 return (irqno); 3515 } 3516 3517 } else { 3518 /* default configuration */ 3519 ioapicindex = 0; 3520 ioapic = apic_io_id[ioapicindex]; 3521 ipin = (uchar_t)irqno; 3522 intr_index = DEFAULT_INDEX; 3523 } 3524 3525 if (ispec == NULL) { 3526 APIC_VERBOSE_IOAPIC((CE_WARN, "No intrspec for irqno = %x\n", 3527 irqno)); 3528 } else if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) { 3529 if ((newirq = apic_share_vector(irqno, intr_flagp, intr_index, 3530 ipl, ioapicindex, ipin, &irqptr)) != -1) { 3531 irqptr->airq_ipl = ipl; 3532 irqptr->airq_origirq = (uchar_t)origirq; 3533 irqptr->airq_dip = dip; 3534 irqptr->airq_major = major; 3535 sdip = apic_irq_table[IRQINDEX(newirq)]->airq_dip; 3536 /* This is OK to do really */ 3537 if (sdip == NULL) { 3538 cmn_err(CE_WARN, "Sharing vectors: %s" 3539 " instance %d and SCI", 3540 ddi_get_name(dip), ddi_get_instance(dip)); 3541 } else { 3542 cmn_err(CE_WARN, "Sharing vectors: %s" 3543 " instance %d and %s instance %d", 3544 ddi_get_name(sdip), ddi_get_instance(sdip), 3545 ddi_get_name(dip), ddi_get_instance(dip)); 3546 } 3547 return (newirq); 3548 } 3549 /* try high priority allocation now that share has failed */ 3550 if ((vector = apic_allocate_vector(ipl, irqno, 1)) == 0) { 3551 cmn_err(CE_WARN, "No interrupt vector: %s instance %d", 3552 ddi_get_name(dip), ddi_get_instance(dip)); 3553 return (-1); 3554 } 3555 } 3556 3557 mutex_enter(&airq_mutex); 3558 if (apic_irq_table[irqno] == NULL) { 3559 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP); 3560 irqptr->airq_temp_cpu = IRQ_UNINIT; 3561 apic_irq_table[irqno] = irqptr; 3562 } else { 3563 irqptr = apic_irq_table[irqno]; 3564 if (irqptr->airq_mps_intr_index != FREE_INDEX) { 3565 /* 3566 * The slot is used by another irqno, so allocate 3567 * a free irqno for this interrupt 3568 */ 3569 newirq = apic_allocate_irq(APIC_FIRST_FREE_IRQ); 3570 if (newirq == -1) { 3571 mutex_exit(&airq_mutex); 3572 return (-1); 3573 } 3574 irqno = newirq; 3575 irqptr = apic_irq_table[irqno]; 3576 if (irqptr == NULL) { 3577 irqptr = kmem_zalloc(sizeof (apic_irq_t), 3578 KM_SLEEP); 3579 irqptr->airq_temp_cpu = IRQ_UNINIT; 3580 apic_irq_table[irqno] = irqptr; 3581 } 3582 apic_modify_vector(vector, newirq); 3583 } 3584 } 3585 apic_max_device_irq = max(irqno, apic_max_device_irq); 3586 apic_min_device_irq = min(irqno, apic_min_device_irq); 3587 mutex_exit(&airq_mutex); 3588 irqptr->airq_ioapicindex = ioapicindex; 3589 irqptr->airq_intin_no = ipin; 3590 irqptr->airq_ipl = ipl; 3591 irqptr->airq_vector = vector; 3592 irqptr->airq_origirq = (uchar_t)origirq; 3593 irqptr->airq_share_id = 0; 3594 irqptr->airq_mps_intr_index = (short)intr_index; 3595 irqptr->airq_dip = dip; 3596 irqptr->airq_major = major; 3597 irqptr->airq_cpu = apic_bind_intr(dip, irqno, ioapic, ipin); 3598 if (intr_flagp) 3599 irqptr->airq_iflag = *intr_flagp; 3600 3601 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) { 3602 /* setup I/O APIC entry for non-MSI/X interrupts */ 3603 apic_record_rdt_entry(irqptr, irqno); 3604 } 3605 return (irqno); 3606 } 3607 3608 /* 3609 * return the cpu to which this intr should be bound. 3610 * Check properties or any other mechanism to see if user wants it 3611 * bound to a specific CPU. If so, return the cpu id with high bit set. 3612 * If not, use the policy to choose a cpu and return the id. 3613 */ 3614 uchar_t 3615 apic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, uchar_t intin) 3616 { 3617 int instance, instno, prop_len, bind_cpu, count; 3618 uint_t i, rc; 3619 uchar_t cpu; 3620 major_t major; 3621 char *name, *drv_name, *prop_val, *cptr; 3622 char prop_name[32]; 3623 3624 3625 if (apic_intr_policy == INTR_LOWEST_PRIORITY) 3626 return (IRQ_UNBOUND); 3627 3628 drv_name = NULL; 3629 rc = DDI_PROP_NOT_FOUND; 3630 major = (major_t)-1; 3631 if (dip != NULL) { 3632 name = ddi_get_name(dip); 3633 major = ddi_name_to_major(name); 3634 drv_name = ddi_major_to_name(major); 3635 instance = ddi_get_instance(dip); 3636 if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) { 3637 i = apic_min_device_irq; 3638 for (; i <= apic_max_device_irq; i++) { 3639 3640 if ((i == irq) || (apic_irq_table[i] == NULL) || 3641 (apic_irq_table[i]->airq_mps_intr_index 3642 == FREE_INDEX)) 3643 continue; 3644 3645 if ((apic_irq_table[i]->airq_major == major) && 3646 (!(apic_irq_table[i]->airq_cpu & 3647 IRQ_USER_BOUND))) { 3648 3649 cpu = apic_irq_table[i]->airq_cpu; 3650 3651 cmn_err(CE_CONT, 3652 "!pcplusmp: %s (%s) instance #%d " 3653 "vector 0x%x ioapic 0x%x " 3654 "intin 0x%x is bound to cpu %d\n", 3655 name, drv_name, instance, irq, 3656 ioapicid, intin, cpu); 3657 return (cpu); 3658 } 3659 } 3660 } 3661 /* 3662 * search for "drvname"_intpt_bind_cpus property first, the 3663 * syntax of the property should be "a[,b,c,...]" where 3664 * instance 0 binds to cpu a, instance 1 binds to cpu b, 3665 * instance 3 binds to cpu c... 3666 * ddi_getlongprop() will search /option first, then / 3667 * if "drvname"_intpt_bind_cpus doesn't exist, then find 3668 * intpt_bind_cpus property. The syntax is the same, and 3669 * it applies to all the devices if its "drvname" specific 3670 * property doesn't exist 3671 */ 3672 (void) strcpy(prop_name, drv_name); 3673 (void) strcat(prop_name, "_intpt_bind_cpus"); 3674 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name, 3675 (caddr_t)&prop_val, &prop_len); 3676 if (rc != DDI_PROP_SUCCESS) { 3677 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, 3678 "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len); 3679 } 3680 } 3681 if (rc == DDI_PROP_SUCCESS) { 3682 for (i = count = 0; i < (prop_len - 1); i++) 3683 if (prop_val[i] == ',') 3684 count++; 3685 if (prop_val[i-1] != ',') 3686 count++; 3687 /* 3688 * if somehow the binding instances defined in the 3689 * property are not enough for this instno., then 3690 * reuse the pattern for the next instance until 3691 * it reaches the requested instno 3692 */ 3693 instno = instance % count; 3694 i = 0; 3695 cptr = prop_val; 3696 while (i < instno) 3697 if (*cptr++ == ',') 3698 i++; 3699 bind_cpu = stoi(&cptr); 3700 kmem_free(prop_val, prop_len); 3701 /* if specific cpu is bogus, then default to cpu 0 */ 3702 if (bind_cpu >= apic_nproc) { 3703 cmn_err(CE_WARN, "pcplusmp: %s=%s: CPU %d not present", 3704 prop_name, prop_val, bind_cpu); 3705 bind_cpu = 0; 3706 } else { 3707 /* indicate that we are bound at user request */ 3708 bind_cpu |= IRQ_USER_BOUND; 3709 } 3710 /* 3711 * no need to check apic_cpus[].aci_status, if specific cpu is 3712 * not up, then post_cpu_start will handle it. 3713 */ 3714 } else { 3715 /* 3716 * We change bind_cpu only for every two calls 3717 * as most drivers still do 2 add_intrs for every 3718 * interrupt 3719 */ 3720 bind_cpu = (apic_next_bind_cpu++) / 2; 3721 if (bind_cpu >= apic_nproc) { 3722 apic_next_bind_cpu = 1; 3723 bind_cpu = 0; 3724 } 3725 } 3726 if (drv_name != NULL) 3727 cmn_err(CE_CONT, "!pcplusmp: %s (%s) instance %d " 3728 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n", 3729 name, drv_name, instance, 3730 irq, ioapicid, intin, bind_cpu & ~IRQ_USER_BOUND); 3731 else 3732 cmn_err(CE_CONT, "!pcplusmp: " 3733 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n", 3734 irq, ioapicid, intin, bind_cpu & ~IRQ_USER_BOUND); 3735 3736 return ((uchar_t)bind_cpu); 3737 } 3738 3739 static struct apic_io_intr * 3740 apic_find_io_intr_w_busid(int irqno, int busid) 3741 { 3742 struct apic_io_intr *intrp; 3743 3744 /* 3745 * It can have more than 1 entry with same source bus IRQ, 3746 * but unique with the source bus id 3747 */ 3748 intrp = apic_io_intrp; 3749 if (intrp != NULL) { 3750 while (intrp->intr_entry == APIC_IO_INTR_ENTRY) { 3751 if (intrp->intr_irq == irqno && 3752 intrp->intr_busid == busid && 3753 intrp->intr_type == IO_INTR_INT) 3754 return (intrp); 3755 intrp++; 3756 } 3757 } 3758 APIC_VERBOSE_IOAPIC((CE_NOTE, "Did not find io intr for irqno:" 3759 "busid %x:%x\n", irqno, busid)); 3760 return ((struct apic_io_intr *)NULL); 3761 } 3762 3763 3764 struct mps_bus_info { 3765 char *bus_name; 3766 int bus_id; 3767 } bus_info_array[] = { 3768 "ISA ", BUS_ISA, 3769 "PCI ", BUS_PCI, 3770 "EISA ", BUS_EISA, 3771 "XPRESS", BUS_XPRESS, 3772 "PCMCIA", BUS_PCMCIA, 3773 "VL ", BUS_VL, 3774 "CBUS ", BUS_CBUS, 3775 "CBUSII", BUS_CBUSII, 3776 "FUTURE", BUS_FUTURE, 3777 "INTERN", BUS_INTERN, 3778 "MBI ", BUS_MBI, 3779 "MBII ", BUS_MBII, 3780 "MPI ", BUS_MPI, 3781 "MPSA ", BUS_MPSA, 3782 "NUBUS ", BUS_NUBUS, 3783 "TC ", BUS_TC, 3784 "VME ", BUS_VME 3785 }; 3786 3787 static int 3788 apic_find_bus_type(char *bus) 3789 { 3790 int i = 0; 3791 3792 for (; i < sizeof (bus_info_array)/sizeof (struct mps_bus_info); i++) 3793 if (strncmp(bus, bus_info_array[i].bus_name, 3794 strlen(bus_info_array[i].bus_name)) == 0) 3795 return (bus_info_array[i].bus_id); 3796 APIC_VERBOSE_IOAPIC((CE_WARN, "Did not find bus type for bus %s", bus)); 3797 return (0); 3798 } 3799 3800 static int 3801 apic_find_bus(int busid) 3802 { 3803 struct apic_bus *busp; 3804 3805 busp = apic_busp; 3806 while (busp->bus_entry == APIC_BUS_ENTRY) { 3807 if (busp->bus_id == busid) 3808 return (apic_find_bus_type((char *)&busp->bus_str1)); 3809 busp++; 3810 } 3811 APIC_VERBOSE_IOAPIC((CE_WARN, "Did not find bus for bus id %x", busid)); 3812 return (0); 3813 } 3814 3815 static int 3816 apic_find_bus_id(int bustype) 3817 { 3818 struct apic_bus *busp; 3819 3820 busp = apic_busp; 3821 while (busp->bus_entry == APIC_BUS_ENTRY) { 3822 if (apic_find_bus_type((char *)&busp->bus_str1) == bustype) 3823 return (busp->bus_id); 3824 busp++; 3825 } 3826 APIC_VERBOSE_IOAPIC((CE_WARN, "Did not find bus id for bustype %x", 3827 bustype)); 3828 return (-1); 3829 } 3830 3831 /* 3832 * Check if a particular irq need to be reserved for any io_intr 3833 */ 3834 static struct apic_io_intr * 3835 apic_find_io_intr(int irqno) 3836 { 3837 struct apic_io_intr *intrp; 3838 3839 intrp = apic_io_intrp; 3840 if (intrp != NULL) { 3841 while (intrp->intr_entry == APIC_IO_INTR_ENTRY) { 3842 if (intrp->intr_irq == irqno && 3843 intrp->intr_type == IO_INTR_INT) 3844 return (intrp); 3845 intrp++; 3846 } 3847 } 3848 return ((struct apic_io_intr *)NULL); 3849 } 3850 3851 /* 3852 * Check if the given ioapicindex intin combination has already been assigned 3853 * an irq. If so return irqno. Else -1 3854 */ 3855 static int 3856 apic_find_intin(uchar_t ioapic, uchar_t intin) 3857 { 3858 apic_irq_t *irqptr; 3859 int i; 3860 3861 /* find ioapic and intin in the apic_irq_table[] and return the index */ 3862 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) { 3863 irqptr = apic_irq_table[i]; 3864 while (irqptr) { 3865 if ((irqptr->airq_mps_intr_index >= 0) && 3866 (irqptr->airq_intin_no == intin) && 3867 (irqptr->airq_ioapicindex == ioapic)) { 3868 APIC_VERBOSE_IOAPIC((CE_NOTE, "!Found irq " 3869 "entry for ioapic:intin %x:%x " 3870 "shared interrupts ?", ioapic, intin)); 3871 return (i); 3872 } 3873 irqptr = irqptr->airq_next; 3874 } 3875 } 3876 return (-1); 3877 } 3878 3879 int 3880 apic_allocate_irq(int irq) 3881 { 3882 int freeirq, i; 3883 3884 if ((freeirq = apic_find_free_irq(irq, (APIC_RESV_IRQ - 1))) == -1) 3885 if ((freeirq = apic_find_free_irq(APIC_FIRST_FREE_IRQ, 3886 (irq - 1))) == -1) { 3887 /* 3888 * if BIOS really defines every single irq in the mps 3889 * table, then don't worry about conflicting with 3890 * them, just use any free slot in apic_irq_table 3891 */ 3892 for (i = APIC_FIRST_FREE_IRQ; i < APIC_RESV_IRQ; i++) { 3893 if ((apic_irq_table[i] == NULL) || 3894 apic_irq_table[i]->airq_mps_intr_index == 3895 FREE_INDEX) { 3896 freeirq = i; 3897 break; 3898 } 3899 } 3900 if (freeirq == -1) { 3901 /* This shouldn't happen, but just in case */ 3902 cmn_err(CE_WARN, "pcplusmp: NO available IRQ"); 3903 return (-1); 3904 } 3905 } 3906 if (apic_irq_table[freeirq] == NULL) { 3907 apic_irq_table[freeirq] = 3908 kmem_zalloc(sizeof (apic_irq_t), KM_NOSLEEP); 3909 if (apic_irq_table[freeirq] == NULL) { 3910 cmn_err(CE_WARN, "pcplusmp: NO memory to allocate IRQ"); 3911 return (-1); 3912 } 3913 apic_irq_table[freeirq]->airq_mps_intr_index = FREE_INDEX; 3914 } 3915 return (freeirq); 3916 } 3917 3918 static int 3919 apic_find_free_irq(int start, int end) 3920 { 3921 int i; 3922 3923 for (i = start; i <= end; i++) 3924 /* Check if any I/O entry needs this IRQ */ 3925 if (apic_find_io_intr(i) == NULL) { 3926 /* Then see if it is free */ 3927 if ((apic_irq_table[i] == NULL) || 3928 (apic_irq_table[i]->airq_mps_intr_index == 3929 FREE_INDEX)) { 3930 return (i); 3931 } 3932 } 3933 return (-1); 3934 } 3935 3936 /* 3937 * Allocate a free vector for irq at ipl. Takes care of merging of multiple 3938 * IPLs into a single APIC level as well as stretching some IPLs onto multiple 3939 * levels. APIC_HI_PRI_VECTS interrupts are reserved for high priority 3940 * requests and allocated only when pri is set. 3941 */ 3942 static uchar_t 3943 apic_allocate_vector(int ipl, int irq, int pri) 3944 { 3945 int lowest, highest, i; 3946 3947 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK; 3948 lowest = apic_ipltopri[ipl - 1] + APIC_VECTOR_PER_IPL; 3949 3950 if (highest < lowest) /* Both ipl and ipl - 1 map to same pri */ 3951 lowest -= APIC_VECTOR_PER_IPL; 3952 3953 #ifdef DEBUG 3954 if (apic_restrict_vector) /* for testing shared interrupt logic */ 3955 highest = lowest + apic_restrict_vector + APIC_HI_PRI_VECTS; 3956 #endif /* DEBUG */ 3957 if (pri == 0) 3958 highest -= APIC_HI_PRI_VECTS; 3959 3960 for (i = lowest; i < highest; i++) { 3961 if ((i == T_FASTTRAP) || (i == APIC_SPUR_INTR) || 3962 (i == T_SYSCALLINT) || (i == T_DTRACE_PROBE) || 3963 (i == T_DTRACE_RET)) 3964 continue; 3965 if (apic_vector_to_irq[i] == APIC_RESV_IRQ) { 3966 apic_vector_to_irq[i] = (uchar_t)irq; 3967 return (i); 3968 } 3969 } 3970 3971 return (0); 3972 } 3973 3974 static void 3975 apic_modify_vector(uchar_t vector, int irq) 3976 { 3977 apic_vector_to_irq[vector] = (uchar_t)irq; 3978 } 3979 3980 /* 3981 * Mark vector as being in the process of being deleted. Interrupts 3982 * may still come in on some CPU. The moment an interrupt comes with 3983 * the new vector, we know we can free the old one. Called only from 3984 * addspl and delspl with interrupts disabled. Because an interrupt 3985 * can be shared, but no interrupt from either device may come in, 3986 * we also use a timeout mechanism, which we arbitrarily set to 3987 * apic_revector_timeout microseconds. 3988 */ 3989 static void 3990 apic_mark_vector(uchar_t oldvector, uchar_t newvector) 3991 { 3992 int iflag = intr_clear(); 3993 lock_set(&apic_revector_lock); 3994 if (!apic_oldvec_to_newvec) { 3995 apic_oldvec_to_newvec = 3996 kmem_zalloc(sizeof (newvector) * APIC_MAX_VECTOR * 2, 3997 KM_NOSLEEP); 3998 3999 if (!apic_oldvec_to_newvec) { 4000 /* 4001 * This failure is not catastrophic. 4002 * But, the oldvec will never be freed. 4003 */ 4004 apic_error |= APIC_ERR_MARK_VECTOR_FAIL; 4005 lock_clear(&apic_revector_lock); 4006 intr_restore(iflag); 4007 return; 4008 } 4009 apic_newvec_to_oldvec = &apic_oldvec_to_newvec[APIC_MAX_VECTOR]; 4010 } 4011 4012 /* See if we already did this for drivers which do double addintrs */ 4013 if (apic_oldvec_to_newvec[oldvector] != newvector) { 4014 apic_oldvec_to_newvec[oldvector] = newvector; 4015 apic_newvec_to_oldvec[newvector] = oldvector; 4016 apic_revector_pending++; 4017 } 4018 lock_clear(&apic_revector_lock); 4019 intr_restore(iflag); 4020 (void) timeout(apic_xlate_vector_free_timeout_handler, 4021 (void *)(uintptr_t)oldvector, drv_usectohz(apic_revector_timeout)); 4022 } 4023 4024 /* 4025 * xlate_vector is called from intr_enter if revector_pending is set. 4026 * It will xlate it if needed and mark the old vector as free. 4027 */ 4028 static uchar_t 4029 apic_xlate_vector(uchar_t vector) 4030 { 4031 uchar_t newvector, oldvector = 0; 4032 4033 lock_set(&apic_revector_lock); 4034 /* Do we really need to do this ? */ 4035 if (!apic_revector_pending) { 4036 lock_clear(&apic_revector_lock); 4037 return (vector); 4038 } 4039 if ((newvector = apic_oldvec_to_newvec[vector]) != 0) 4040 oldvector = vector; 4041 else { 4042 /* 4043 * The incoming vector is new . See if a stale entry is 4044 * remaining 4045 */ 4046 if ((oldvector = apic_newvec_to_oldvec[vector]) != 0) 4047 newvector = vector; 4048 } 4049 4050 if (oldvector) { 4051 apic_revector_pending--; 4052 apic_oldvec_to_newvec[oldvector] = 0; 4053 apic_newvec_to_oldvec[newvector] = 0; 4054 apic_free_vector(oldvector); 4055 lock_clear(&apic_revector_lock); 4056 /* There could have been more than one reprogramming! */ 4057 return (apic_xlate_vector(newvector)); 4058 } 4059 lock_clear(&apic_revector_lock); 4060 return (vector); 4061 } 4062 4063 void 4064 apic_xlate_vector_free_timeout_handler(void *arg) 4065 { 4066 int iflag; 4067 uchar_t oldvector, newvector; 4068 4069 oldvector = (uchar_t)(uintptr_t)arg; 4070 iflag = intr_clear(); 4071 lock_set(&apic_revector_lock); 4072 if ((newvector = apic_oldvec_to_newvec[oldvector]) != 0) { 4073 apic_free_vector(oldvector); 4074 apic_oldvec_to_newvec[oldvector] = 0; 4075 apic_newvec_to_oldvec[newvector] = 0; 4076 apic_revector_pending--; 4077 } 4078 4079 lock_clear(&apic_revector_lock); 4080 intr_restore(iflag); 4081 } 4082 4083 4084 /* Mark vector as not being used by any irq */ 4085 static void 4086 apic_free_vector(uchar_t vector) 4087 { 4088 apic_vector_to_irq[vector] = APIC_RESV_IRQ; 4089 } 4090 4091 /* 4092 * compute the polarity, trigger mode and vector for programming into 4093 * the I/O apic and record in airq_rdt_entry. 4094 */ 4095 static void 4096 apic_record_rdt_entry(apic_irq_t *irqptr, int irq) 4097 { 4098 int ioapicindex, bus_type, vector; 4099 short intr_index; 4100 uint_t level, po, io_po; 4101 struct apic_io_intr *iointrp; 4102 4103 intr_index = irqptr->airq_mps_intr_index; 4104 DDI_INTR_IMPLDBG((CE_CONT, "apic_record_rdt_entry: intr_index=%d " 4105 "irq = 0x%x dip = 0x%p vector = 0x%x\n", intr_index, irq, 4106 (void *)irqptr->airq_dip, irqptr->airq_vector)); 4107 4108 if (intr_index == RESERVE_INDEX) { 4109 apic_error |= APIC_ERR_INVALID_INDEX; 4110 return; 4111 } else if (APIC_IS_MSI_OR_MSIX_INDEX(intr_index)) { 4112 return; 4113 } 4114 4115 vector = irqptr->airq_vector; 4116 ioapicindex = irqptr->airq_ioapicindex; 4117 /* Assume edge triggered by default */ 4118 level = 0; 4119 /* Assume active high by default */ 4120 po = 0; 4121 4122 if (intr_index == DEFAULT_INDEX || intr_index == FREE_INDEX) { 4123 ASSERT(irq < 16); 4124 if (eisa_level_intr_mask & (1 << irq)) 4125 level = AV_LEVEL; 4126 if (intr_index == FREE_INDEX && apic_defconf == 0) 4127 apic_error |= APIC_ERR_INVALID_INDEX; 4128 } else if (intr_index == ACPI_INDEX) { 4129 bus_type = irqptr->airq_iflag.bustype; 4130 if (irqptr->airq_iflag.intr_el == INTR_EL_CONFORM) { 4131 if (bus_type == BUS_PCI) 4132 level = AV_LEVEL; 4133 } else 4134 level = (irqptr->airq_iflag.intr_el == INTR_EL_LEVEL) ? 4135 AV_LEVEL : 0; 4136 if (level && 4137 ((irqptr->airq_iflag.intr_po == INTR_PO_ACTIVE_LOW) || 4138 (irqptr->airq_iflag.intr_po == INTR_PO_CONFORM && 4139 bus_type == BUS_PCI))) 4140 po = AV_ACTIVE_LOW; 4141 } else { 4142 iointrp = apic_io_intrp + intr_index; 4143 bus_type = apic_find_bus(iointrp->intr_busid); 4144 if (iointrp->intr_el == INTR_EL_CONFORM) { 4145 if ((irq < 16) && (eisa_level_intr_mask & (1 << irq))) 4146 level = AV_LEVEL; 4147 else if (bus_type == BUS_PCI) 4148 level = AV_LEVEL; 4149 } else 4150 level = (iointrp->intr_el == INTR_EL_LEVEL) ? 4151 AV_LEVEL : 0; 4152 if (level && ((iointrp->intr_po == INTR_PO_ACTIVE_LOW) || 4153 (iointrp->intr_po == INTR_PO_CONFORM && 4154 bus_type == BUS_PCI))) 4155 po = AV_ACTIVE_LOW; 4156 } 4157 if (level) 4158 apic_level_intr[irq] = 1; 4159 /* 4160 * The 82489DX External APIC cannot do active low polarity interrupts. 4161 */ 4162 if (po && (apic_io_ver[ioapicindex] != IOAPIC_VER_82489DX)) 4163 io_po = po; 4164 else 4165 io_po = 0; 4166 4167 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) 4168 printf("setio: ioapic=%x intin=%x level=%x po=%x vector=%x\n", 4169 ioapicindex, irqptr->airq_intin_no, level, io_po, vector); 4170 4171 irqptr->airq_rdt_entry = level|io_po|vector; 4172 } 4173 4174 /* 4175 * Call rebind to do the actual programming. 4176 */ 4177 static int 4178 apic_setup_io_intr(apic_irq_t *irqptr, int irq) 4179 { 4180 int rv; 4181 4182 if (rv = apic_rebind(irqptr, apic_irq_table[irq]->airq_cpu, 1, 4183 IMMEDIATE)) 4184 /* CPU is not up or interrupt is disabled. Fall back to 0 */ 4185 rv = apic_rebind(irqptr, 0, 1, IMMEDIATE); 4186 4187 return (rv); 4188 } 4189 4190 /* 4191 * Deferred reprogramming: Call apic_rebind to do the real work. 4192 */ 4193 static int 4194 apic_setup_io_intr_deferred(apic_irq_t *irqptr, int irq) 4195 { 4196 int rv; 4197 4198 if (rv = apic_rebind(irqptr, apic_irq_table[irq]->airq_cpu, 1, 4199 DEFERRED)) 4200 /* CPU is not up or interrupt is disabled. Fall back to 0 */ 4201 rv = apic_rebind(irqptr, 0, 1, DEFERRED); 4202 4203 return (rv); 4204 } 4205 4206 /* 4207 * Bind interrupt corresponding to irq_ptr to bind_cpu. acquire_lock 4208 * if false (0) means lock is already held (e.g: in rebind_all). 4209 */ 4210 static int 4211 apic_rebind(apic_irq_t *irq_ptr, int bind_cpu, int acquire_lock, int when) 4212 { 4213 int intin_no; 4214 volatile int32_t *ioapic; 4215 uchar_t airq_temp_cpu; 4216 apic_cpus_info_t *cpu_infop; 4217 int iflag; 4218 int which_irq = apic_vector_to_irq[irq_ptr->airq_vector]; 4219 4220 intin_no = irq_ptr->airq_intin_no; 4221 ioapic = apicioadr[irq_ptr->airq_ioapicindex]; 4222 airq_temp_cpu = irq_ptr->airq_temp_cpu; 4223 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) { 4224 if (airq_temp_cpu & IRQ_USER_BOUND) 4225 /* Mask off high bit so it can be used as array index */ 4226 airq_temp_cpu &= ~IRQ_USER_BOUND; 4227 4228 ASSERT(airq_temp_cpu < apic_nproc); 4229 } 4230 4231 iflag = intr_clear(); 4232 4233 if (acquire_lock) 4234 lock_set(&apic_ioapic_lock); 4235 4236 /* 4237 * Can't bind to a CPU that's not online: 4238 */ 4239 cpu_infop = &apic_cpus[bind_cpu & ~IRQ_USER_BOUND]; 4240 if (!(cpu_infop->aci_status & APIC_CPU_INTR_ENABLE)) { 4241 4242 if (acquire_lock) 4243 lock_clear(&apic_ioapic_lock); 4244 4245 intr_restore(iflag); 4246 return (1); 4247 } 4248 4249 /* 4250 * If this is a deferred reprogramming attempt, ensure we have 4251 * not been passed stale data: 4252 */ 4253 if ((when == DEFERRED) && 4254 (apic_reprogram_info[which_irq].valid == 0)) { 4255 /* stale info, so just return */ 4256 if (acquire_lock) 4257 lock_clear(&apic_ioapic_lock); 4258 4259 intr_restore(iflag); 4260 return (0); 4261 } 4262 4263 /* 4264 * If this interrupt has been delivered to a CPU and that CPU 4265 * has not handled it yet, we cannot reprogram the IOAPIC now: 4266 */ 4267 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index) && 4268 apic_check_stuck_interrupt(irq_ptr, airq_temp_cpu, bind_cpu, 4269 ioapic, intin_no, which_irq) != 0) { 4270 4271 if (acquire_lock) 4272 lock_clear(&apic_ioapic_lock); 4273 4274 intr_restore(iflag); 4275 return (0); 4276 } 4277 4278 /* 4279 * NOTE: We do not unmask the RDT here, as an interrupt MAY still 4280 * come in before we have a chance to reprogram it below. The 4281 * reprogramming below will simultaneously change and unmask the 4282 * RDT entry. 4283 */ 4284 4285 if ((uchar_t)bind_cpu == IRQ_UNBOUND) { 4286 /* Write the RDT entry -- no specific CPU binding */ 4287 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapic, intin_no, AV_TOALL); 4288 4289 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) 4290 apic_cpus[airq_temp_cpu].aci_temp_bound--; 4291 4292 /* Write the vector, trigger, and polarity portion of the RDT */ 4293 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no, 4294 AV_LDEST | AV_LOPRI | irq_ptr->airq_rdt_entry); 4295 if (acquire_lock) 4296 lock_clear(&apic_ioapic_lock); 4297 irq_ptr->airq_temp_cpu = IRQ_UNBOUND; 4298 intr_restore(iflag); 4299 return (0); 4300 } 4301 4302 if (bind_cpu & IRQ_USER_BOUND) { 4303 cpu_infop->aci_bound++; 4304 } else { 4305 cpu_infop->aci_temp_bound++; 4306 } 4307 ASSERT((bind_cpu & ~IRQ_USER_BOUND) < apic_nproc); 4308 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) { 4309 /* Write the RDT entry -- bind to a specific CPU: */ 4310 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapic, intin_no, 4311 cpu_infop->aci_local_id << APIC_ID_BIT_OFFSET); 4312 } 4313 if ((airq_temp_cpu != IRQ_UNBOUND) && (airq_temp_cpu != IRQ_UNINIT)) { 4314 apic_cpus[airq_temp_cpu].aci_temp_bound--; 4315 } 4316 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) { 4317 /* Write the vector, trigger, and polarity portion of the RDT */ 4318 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no, 4319 AV_PDEST | AV_FIXED | irq_ptr->airq_rdt_entry); 4320 } else { 4321 if (irq_ptr->airq_ioapicindex == irq_ptr->airq_origirq) { 4322 /* first one */ 4323 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call " 4324 "apic_pci_msi_enable_vector\n")); 4325 if (apic_pci_msi_enable_vector(irq_ptr->airq_dip, 4326 (irq_ptr->airq_mps_intr_index == MSI_INDEX) ? 4327 DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX, which_irq, 4328 irq_ptr->airq_vector, irq_ptr->airq_intin_no, 4329 cpu_infop->aci_local_id) != PSM_SUCCESS) { 4330 cmn_err(CE_WARN, "pcplusmp: " 4331 "apic_pci_msi_enable_vector " 4332 "returned PSM_FAILURE"); 4333 } 4334 } 4335 if ((irq_ptr->airq_ioapicindex + irq_ptr->airq_intin_no - 1) == 4336 irq_ptr->airq_origirq) { /* last one */ 4337 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call " 4338 "pci_msi_enable_mode\n")); 4339 if (pci_msi_enable_mode(irq_ptr->airq_dip, 4340 (irq_ptr->airq_mps_intr_index == MSI_INDEX) ? 4341 DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX, 4342 which_irq) != DDI_SUCCESS) { 4343 DDI_INTR_IMPLDBG((CE_CONT, "pcplusmp: " 4344 "pci_msi_enable failed\n")); 4345 (void) pci_msi_unconfigure(irq_ptr->airq_dip, 4346 (irq_ptr->airq_mps_intr_index == MSI_INDEX) ? 4347 DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX, 4348 which_irq); 4349 } 4350 } 4351 } 4352 if (acquire_lock) 4353 lock_clear(&apic_ioapic_lock); 4354 irq_ptr->airq_temp_cpu = (uchar_t)bind_cpu; 4355 apic_redist_cpu_skip &= ~(1 << (bind_cpu & ~IRQ_USER_BOUND)); 4356 intr_restore(iflag); 4357 return (0); 4358 } 4359 4360 /* 4361 * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR 4362 * bit set. Sets up a timeout to perform the reprogramming at a later time 4363 * if it cannot wait for the Remote IRR bit to clear (or if waiting did not 4364 * result in the bit's clearing). 4365 * 4366 * This function will mask the RDT entry if the Remote IRR bit is set. 4367 * 4368 * Returns non-zero if the caller should defer IOAPIC reprogramming. 4369 */ 4370 static int 4371 apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu, 4372 int new_bind_cpu, volatile int32_t *ioapic, int intin_no, int which_irq) 4373 { 4374 int32_t rdt_entry; 4375 int waited; 4376 4377 /* Mask the RDT entry, but only if it's a level-triggered interrupt */ 4378 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no); 4379 if ((rdt_entry & (AV_LEVEL|AV_MASK)) == AV_LEVEL) { 4380 4381 /* Mask it */ 4382 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no, 4383 AV_MASK | rdt_entry); 4384 } 4385 4386 /* 4387 * Wait for the delivery pending bit to clear. 4388 */ 4389 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no) & 4390 (AV_LEVEL|AV_PENDING)) == (AV_LEVEL|AV_PENDING)) { 4391 4392 /* 4393 * If we're still waiting on the delivery of this interrupt, 4394 * continue to wait here until it is delivered (this should be 4395 * a very small amount of time, but include a timeout just in 4396 * case). 4397 */ 4398 for (waited = 0; waited < apic_max_usecs_clear_pending; 4399 waited += APIC_USECS_PER_WAIT_INTERVAL) { 4400 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no) 4401 & AV_PENDING) == 0) { 4402 break; 4403 } 4404 drv_usecwait(APIC_USECS_PER_WAIT_INTERVAL); 4405 } 4406 4407 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no) & 4408 AV_PENDING) != 0) { 4409 cmn_err(CE_WARN, "!IOAPIC %d intin %d: Could not " 4410 "deliver interrupt to local APIC within " 4411 "%d usecs.", irq_ptr->airq_ioapicindex, 4412 irq_ptr->airq_intin_no, 4413 apic_max_usecs_clear_pending); 4414 } 4415 } 4416 4417 /* 4418 * If the remote IRR bit is set, then the interrupt has been sent 4419 * to a CPU for processing. We have no choice but to wait for 4420 * that CPU to process the interrupt, at which point the remote IRR 4421 * bit will be cleared. 4422 */ 4423 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no) & 4424 (AV_LEVEL|AV_REMOTE_IRR)) == (AV_LEVEL|AV_REMOTE_IRR)) { 4425 4426 /* 4427 * If the CPU that this RDT is bound to is NOT the current 4428 * CPU, wait until that CPU handles the interrupt and ACKs 4429 * it. If this interrupt is not bound to any CPU (that is, 4430 * if it's bound to the logical destination of "anyone"), it 4431 * may have been delivered to the current CPU so handle that 4432 * case by deferring the reprogramming (below). 4433 */ 4434 kpreempt_disable(); 4435 if ((old_bind_cpu != IRQ_UNBOUND) && 4436 (old_bind_cpu != IRQ_UNINIT) && 4437 (old_bind_cpu != psm_get_cpu_id())) { 4438 for (waited = 0; waited < apic_max_usecs_clear_pending; 4439 waited += APIC_USECS_PER_WAIT_INTERVAL) { 4440 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, 4441 intin_no) & AV_REMOTE_IRR) == 0) { 4442 4443 /* Clear the reprogramming state: */ 4444 lock_set(&apic_ioapic_reprogram_lock); 4445 4446 apic_reprogram_info[which_irq].valid 4447 = 0; 4448 apic_reprogram_info[which_irq].bindcpu 4449 = 0; 4450 apic_reprogram_info[which_irq].timeouts 4451 = 0; 4452 4453 lock_clear(&apic_ioapic_reprogram_lock); 4454 4455 /* Remote IRR has cleared! */ 4456 kpreempt_enable(); 4457 return (0); 4458 } 4459 drv_usecwait(APIC_USECS_PER_WAIT_INTERVAL); 4460 } 4461 } 4462 kpreempt_enable(); 4463 4464 /* 4465 * If we waited and the Remote IRR bit is still not cleared, 4466 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS 4467 * times for this interrupt, try the last-ditch workarounds: 4468 */ 4469 if (apic_reprogram_info[which_irq].timeouts >= 4470 APIC_REPROGRAM_MAX_TIMEOUTS) { 4471 4472 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, intin_no) 4473 & AV_REMOTE_IRR) != 0) { 4474 /* 4475 * Trying to clear the bit through normal 4476 * channels has failed. So as a last-ditch 4477 * effort, try to set the trigger mode to 4478 * edge, then to level. This has been 4479 * observed to work on many systems. 4480 */ 4481 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, 4482 intin_no, 4483 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, 4484 intin_no) & ~AV_LEVEL); 4485 4486 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, 4487 intin_no, 4488 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, 4489 intin_no) | AV_LEVEL); 4490 4491 /* 4492 * If the bit's STILL set, declare total and 4493 * utter failure 4494 */ 4495 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic, 4496 intin_no) & AV_REMOTE_IRR) != 0) { 4497 cmn_err(CE_WARN, "!IOAPIC %d intin %d: " 4498 "Remote IRR failed to reset " 4499 "within %d usecs. Interrupts to " 4500 "this pin may cease to function.", 4501 irq_ptr->airq_ioapicindex, 4502 irq_ptr->airq_intin_no, 4503 apic_max_usecs_clear_pending); 4504 } 4505 } 4506 /* Clear the reprogramming state: */ 4507 lock_set(&apic_ioapic_reprogram_lock); 4508 4509 apic_reprogram_info[which_irq].valid = 0; 4510 apic_reprogram_info[which_irq].bindcpu = 0; 4511 apic_reprogram_info[which_irq].timeouts = 0; 4512 4513 lock_clear(&apic_ioapic_reprogram_lock); 4514 } else { 4515 #ifdef DEBUG 4516 cmn_err(CE_WARN, "Deferring reprogramming of irq %d", 4517 which_irq); 4518 #endif /* DEBUG */ 4519 /* 4520 * If waiting for the Remote IRR bit (above) didn't 4521 * allow it to clear, defer the reprogramming: 4522 */ 4523 lock_set(&apic_ioapic_reprogram_lock); 4524 4525 apic_reprogram_info[which_irq].valid = 1; 4526 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu; 4527 apic_reprogram_info[which_irq].timeouts++; 4528 4529 lock_clear(&apic_ioapic_reprogram_lock); 4530 4531 /* Fire up a timeout to handle this later */ 4532 (void) timeout(apic_reprogram_timeout_handler, 4533 (void *) 0, 4534 drv_usectohz(APIC_REPROGRAM_TIMEOUT_DELAY)); 4535 4536 /* Inform caller to defer IOAPIC programming: */ 4537 return (1); 4538 } 4539 } 4540 return (0); 4541 } 4542 4543 /* 4544 * Timeout handler that performs the APIC reprogramming 4545 */ 4546 /*ARGSUSED*/ 4547 static void 4548 apic_reprogram_timeout_handler(void *arg) 4549 { 4550 /*LINTED: set but not used in function*/ 4551 int i, result; 4552 4553 /* Serialize access to this function */ 4554 mutex_enter(&apic_reprogram_timeout_mutex); 4555 4556 /* 4557 * For each entry in the reprogramming state that's valid, 4558 * try the reprogramming again: 4559 */ 4560 for (i = 0; i < APIC_MAX_VECTOR; i++) { 4561 if (apic_reprogram_info[i].valid == 0) 4562 continue; 4563 /* 4564 * Though we can't really do anything about errors 4565 * at this point, keep track of them for reporting. 4566 * Note that it is very possible for apic_setup_io_intr 4567 * to re-register this very timeout if the Remote IRR bit 4568 * has not yet cleared. 4569 */ 4570 result = apic_setup_io_intr_deferred(apic_irq_table[i], i); 4571 4572 #ifdef DEBUG 4573 if (result) 4574 cmn_err(CE_WARN, "apic_reprogram_timeout: " 4575 "apic_setup_io_intr returned nonzero for " 4576 "irq=%d!", i); 4577 #endif /* DEBUG */ 4578 } 4579 4580 mutex_exit(&apic_reprogram_timeout_mutex); 4581 } 4582 4583 4584 /* 4585 * Called to migrate all interrupts at an irq to another cpu. safe 4586 * if true means we are not being called from an interrupt 4587 * context and hence it is safe to do a lock_set. If false 4588 * do only a lock_try and return failure ( non 0 ) if we cannot get it 4589 */ 4590 static int 4591 apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu, int safe) 4592 { 4593 apic_irq_t *irqptr = irq_ptr; 4594 int retval = 0; 4595 int iflag; 4596 4597 iflag = intr_clear(); 4598 if (!safe) { 4599 if (lock_try(&apic_ioapic_lock) == 0) { 4600 intr_restore(iflag); 4601 return (1); 4602 } 4603 } else 4604 lock_set(&apic_ioapic_lock); 4605 4606 while (irqptr) { 4607 if (irqptr->airq_temp_cpu != IRQ_UNINIT) 4608 retval |= apic_rebind(irqptr, bind_cpu, 0, IMMEDIATE); 4609 irqptr = irqptr->airq_next; 4610 } 4611 lock_clear(&apic_ioapic_lock); 4612 intr_restore(iflag); 4613 return (retval); 4614 } 4615 4616 /* 4617 * apic_intr_redistribute does all the messy computations for identifying 4618 * which interrupt to move to which CPU. Currently we do just one interrupt 4619 * at a time. This reduces the time we spent doing all this within clock 4620 * interrupt. When it is done in idle, we could do more than 1. 4621 * First we find the most busy and the most free CPU (time in ISR only) 4622 * skipping those CPUs that has been identified as being ineligible (cpu_skip) 4623 * Then we look for IRQs which are closest to the difference between the 4624 * most busy CPU and the average ISR load. We try to find one whose load 4625 * is less than difference.If none exists, then we chose one larger than the 4626 * difference, provided it does not make the most idle CPU worse than the 4627 * most busy one. In the end, we clear all the busy fields for CPUs. For 4628 * IRQs, they are cleared as they are scanned. 4629 */ 4630 static void 4631 apic_intr_redistribute() 4632 { 4633 int busiest_cpu, most_free_cpu; 4634 int cpu_free, cpu_busy, max_busy, min_busy; 4635 int min_free, diff; 4636 int average_busy, cpus_online; 4637 int i, busy; 4638 apic_cpus_info_t *cpu_infop; 4639 apic_irq_t *min_busy_irq = NULL; 4640 apic_irq_t *max_busy_irq = NULL; 4641 4642 busiest_cpu = most_free_cpu = -1; 4643 cpu_free = cpu_busy = max_busy = average_busy = 0; 4644 min_free = apic_sample_factor_redistribution; 4645 cpus_online = 0; 4646 /* 4647 * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu 4648 * without ioapic_lock. That is OK as we are just doing statistical 4649 * sampling anyway and any inaccuracy now will get corrected next time 4650 * The call to rebind which actually changes things will make sure 4651 * we are consistent. 4652 */ 4653 for (i = 0; i < apic_nproc; i++) { 4654 if (!(apic_redist_cpu_skip & (1 << i)) && 4655 (apic_cpus[i].aci_status & APIC_CPU_INTR_ENABLE)) { 4656 4657 cpu_infop = &apic_cpus[i]; 4658 /* 4659 * If no unbound interrupts or only 1 total on this 4660 * CPU, skip 4661 */ 4662 if (!cpu_infop->aci_temp_bound || 4663 (cpu_infop->aci_bound + cpu_infop->aci_temp_bound) 4664 == 1) { 4665 apic_redist_cpu_skip |= 1 << i; 4666 continue; 4667 } 4668 4669 busy = cpu_infop->aci_busy; 4670 average_busy += busy; 4671 cpus_online++; 4672 if (max_busy < busy) { 4673 max_busy = busy; 4674 busiest_cpu = i; 4675 } 4676 if (min_free > busy) { 4677 min_free = busy; 4678 most_free_cpu = i; 4679 } 4680 if (busy > apic_int_busy_mark) { 4681 cpu_busy |= 1 << i; 4682 } else { 4683 if (busy < apic_int_free_mark) 4684 cpu_free |= 1 << i; 4685 } 4686 } 4687 } 4688 if ((cpu_busy && cpu_free) || 4689 (max_busy >= (min_free + apic_diff_for_redistribution))) { 4690 4691 apic_num_imbalance++; 4692 #ifdef DEBUG 4693 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) { 4694 prom_printf( 4695 "redistribute busy=%x free=%x max=%x min=%x", 4696 cpu_busy, cpu_free, max_busy, min_free); 4697 } 4698 #endif /* DEBUG */ 4699 4700 4701 average_busy /= cpus_online; 4702 4703 diff = max_busy - average_busy; 4704 min_busy = max_busy; /* start with the max possible value */ 4705 max_busy = 0; 4706 min_busy_irq = max_busy_irq = NULL; 4707 i = apic_min_device_irq; 4708 for (; i < apic_max_device_irq; i++) { 4709 apic_irq_t *irq_ptr; 4710 /* Change to linked list per CPU ? */ 4711 if ((irq_ptr = apic_irq_table[i]) == NULL) 4712 continue; 4713 /* Check for irq_busy & decide which one to move */ 4714 /* Also zero them for next round */ 4715 if ((irq_ptr->airq_temp_cpu == busiest_cpu) && 4716 irq_ptr->airq_busy) { 4717 if (irq_ptr->airq_busy < diff) { 4718 /* 4719 * Check for least busy CPU, 4720 * best fit or what ? 4721 */ 4722 if (max_busy < irq_ptr->airq_busy) { 4723 /* 4724 * Most busy within the 4725 * required differential 4726 */ 4727 max_busy = irq_ptr->airq_busy; 4728 max_busy_irq = irq_ptr; 4729 } 4730 } else { 4731 if (min_busy > irq_ptr->airq_busy) { 4732 /* 4733 * least busy, but more than 4734 * the reqd diff 4735 */ 4736 if (min_busy < 4737 (diff + average_busy - 4738 min_free)) { 4739 /* 4740 * Making sure new cpu 4741 * will not end up 4742 * worse 4743 */ 4744 min_busy = 4745 irq_ptr->airq_busy; 4746 4747 min_busy_irq = irq_ptr; 4748 } 4749 } 4750 } 4751 } 4752 irq_ptr->airq_busy = 0; 4753 } 4754 4755 if (max_busy_irq != NULL) { 4756 #ifdef DEBUG 4757 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) { 4758 prom_printf("rebinding %x to %x", 4759 max_busy_irq->airq_vector, most_free_cpu); 4760 } 4761 #endif /* DEBUG */ 4762 if (apic_rebind_all(max_busy_irq, most_free_cpu, 0) 4763 == 0) 4764 /* Make change permenant */ 4765 max_busy_irq->airq_cpu = (uchar_t)most_free_cpu; 4766 } else if (min_busy_irq != NULL) { 4767 #ifdef DEBUG 4768 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) { 4769 prom_printf("rebinding %x to %x", 4770 min_busy_irq->airq_vector, most_free_cpu); 4771 } 4772 #endif /* DEBUG */ 4773 4774 if (apic_rebind_all(min_busy_irq, most_free_cpu, 0) == 4775 0) 4776 /* Make change permenant */ 4777 min_busy_irq->airq_cpu = (uchar_t)most_free_cpu; 4778 } else { 4779 if (cpu_busy != (1 << busiest_cpu)) { 4780 apic_redist_cpu_skip |= 1 << busiest_cpu; 4781 /* 4782 * We leave cpu_skip set so that next time we 4783 * can choose another cpu 4784 */ 4785 } 4786 } 4787 apic_num_rebind++; 4788 } else { 4789 /* 4790 * found nothing. Could be that we skipped over valid CPUs 4791 * or we have balanced everything. If we had a variable 4792 * ticks_for_redistribution, it could be increased here. 4793 * apic_int_busy, int_free etc would also need to be 4794 * changed. 4795 */ 4796 if (apic_redist_cpu_skip) 4797 apic_redist_cpu_skip = 0; 4798 } 4799 for (i = 0; i < apic_nproc; i++) { 4800 apic_cpus[i].aci_busy = 0; 4801 } 4802 } 4803 4804 static void 4805 apic_cleanup_busy() 4806 { 4807 int i; 4808 apic_irq_t *irq_ptr; 4809 4810 for (i = 0; i < apic_nproc; i++) { 4811 apic_cpus[i].aci_busy = 0; 4812 } 4813 4814 for (i = apic_min_device_irq; i < apic_max_device_irq; i++) { 4815 if ((irq_ptr = apic_irq_table[i]) != NULL) 4816 irq_ptr->airq_busy = 0; 4817 } 4818 apic_skipped_redistribute = 0; 4819 } 4820 4821 4822 /* 4823 * This function will reprogram the timer. 4824 * 4825 * When in oneshot mode the argument is the absolute time in future to 4826 * generate the interrupt at. 4827 * 4828 * When in periodic mode, the argument is the interval at which the 4829 * interrupts should be generated. There is no need to support the periodic 4830 * mode timer change at this time. 4831 */ 4832 static void 4833 apic_timer_reprogram(hrtime_t time) 4834 { 4835 hrtime_t now; 4836 uint_t ticks; 4837 4838 /* 4839 * We should be called from high PIL context (CBE_HIGH_PIL), 4840 * so kpreempt is disabled. 4841 */ 4842 4843 if (!apic_oneshot) { 4844 /* time is the interval for periodic mode */ 4845 ticks = (uint_t)((time) / apic_nsec_per_tick); 4846 } else { 4847 /* one shot mode */ 4848 4849 now = gethrtime(); 4850 4851 if (time <= now) { 4852 /* 4853 * requested to generate an interrupt in the past 4854 * generate an interrupt as soon as possible 4855 */ 4856 ticks = apic_min_timer_ticks; 4857 } else if ((time - now) > apic_nsec_max) { 4858 /* 4859 * requested to generate an interrupt at a time 4860 * further than what we are capable of. Set to max 4861 * the hardware can handle 4862 */ 4863 4864 ticks = APIC_MAXVAL; 4865 #ifdef DEBUG 4866 cmn_err(CE_CONT, "apic_timer_reprogram, request at" 4867 " %lld too far in future, current time" 4868 " %lld \n", time, now); 4869 #endif /* DEBUG */ 4870 } else 4871 ticks = (uint_t)((time - now) / apic_nsec_per_tick); 4872 } 4873 4874 if (ticks < apic_min_timer_ticks) 4875 ticks = apic_min_timer_ticks; 4876 4877 apicadr[APIC_INIT_COUNT] = ticks; 4878 4879 } 4880 4881 /* 4882 * This function will enable timer interrupts. 4883 */ 4884 static void 4885 apic_timer_enable(void) 4886 { 4887 /* 4888 * We should be Called from high PIL context (CBE_HIGH_PIL), 4889 * so kpreempt is disabled. 4890 */ 4891 4892 if (!apic_oneshot) 4893 apicadr[APIC_LOCAL_TIMER] = 4894 (apic_clkvect + APIC_BASE_VECT) | AV_TIME; 4895 else { 4896 /* one shot */ 4897 apicadr[APIC_LOCAL_TIMER] = (apic_clkvect + APIC_BASE_VECT); 4898 } 4899 } 4900 4901 /* 4902 * This function will disable timer interrupts. 4903 */ 4904 static void 4905 apic_timer_disable(void) 4906 { 4907 /* 4908 * We should be Called from high PIL context (CBE_HIGH_PIL), 4909 * so kpreempt is disabled. 4910 */ 4911 4912 apicadr[APIC_LOCAL_TIMER] = (apic_clkvect + APIC_BASE_VECT) | AV_MASK; 4913 } 4914 4915 4916 cyclic_id_t apic_cyclic_id; 4917 4918 /* 4919 * If this module needs to be a consumer of cyclic subsystem, they 4920 * can be added here, since at this time kernel cyclic subsystem is initialized 4921 * argument is not currently used, and is reserved for future. 4922 */ 4923 static void 4924 apic_post_cyclic_setup(void *arg) 4925 { 4926 _NOTE(ARGUNUSED(arg)) 4927 cyc_handler_t hdlr; 4928 cyc_time_t when; 4929 4930 /* cpu_lock is held */ 4931 4932 /* set up cyclics for intr redistribution */ 4933 4934 /* 4935 * In peridoc mode intr redistribution processing is done in 4936 * apic_intr_enter during clk intr processing 4937 */ 4938 if (!apic_oneshot) 4939 return; 4940 4941 hdlr.cyh_level = CY_LOW_LEVEL; 4942 hdlr.cyh_func = (cyc_func_t)apic_redistribute_compute; 4943 hdlr.cyh_arg = NULL; 4944 4945 when.cyt_when = 0; 4946 when.cyt_interval = apic_redistribute_sample_interval; 4947 apic_cyclic_id = cyclic_add(&hdlr, &when); 4948 4949 4950 } 4951 4952 static void 4953 apic_redistribute_compute(void) 4954 { 4955 int i, j, max_busy; 4956 4957 if (apic_enable_dynamic_migration) { 4958 if (++apic_nticks == apic_sample_factor_redistribution) { 4959 /* 4960 * Time to call apic_intr_redistribute(). 4961 * reset apic_nticks. This will cause max_busy 4962 * to be calculated below and if it is more than 4963 * apic_int_busy, we will do the whole thing 4964 */ 4965 apic_nticks = 0; 4966 } 4967 max_busy = 0; 4968 for (i = 0; i < apic_nproc; i++) { 4969 4970 /* 4971 * Check if curipl is non zero & if ISR is in 4972 * progress 4973 */ 4974 if (((j = apic_cpus[i].aci_curipl) != 0) && 4975 (apic_cpus[i].aci_ISR_in_progress & (1 << j))) { 4976 4977 int irq; 4978 apic_cpus[i].aci_busy++; 4979 irq = apic_cpus[i].aci_current[j]; 4980 apic_irq_table[irq]->airq_busy++; 4981 } 4982 4983 if (!apic_nticks && 4984 (apic_cpus[i].aci_busy > max_busy)) 4985 max_busy = apic_cpus[i].aci_busy; 4986 } 4987 if (!apic_nticks) { 4988 if (max_busy > apic_int_busy_mark) { 4989 /* 4990 * We could make the following check be 4991 * skipped > 1 in which case, we get a 4992 * redistribution at half the busy mark (due to 4993 * double interval). Need to be able to collect 4994 * more empirical data to decide if that is a 4995 * good strategy. Punt for now. 4996 */ 4997 if (apic_skipped_redistribute) 4998 apic_cleanup_busy(); 4999 else 5000 apic_intr_redistribute(); 5001 } else 5002 apic_skipped_redistribute++; 5003 } 5004 } 5005 } 5006 5007 5008 static int 5009 apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid, 5010 int ipin, int *pci_irqp, iflag_t *intr_flagp) 5011 { 5012 5013 int status; 5014 acpi_psm_lnk_t acpipsmlnk; 5015 5016 if ((status = acpi_get_irq_cache_ent(busid, devid, ipin, pci_irqp, 5017 intr_flagp)) == ACPI_PSM_SUCCESS) { 5018 APIC_VERBOSE_IRQ((CE_CONT, "!pcplusmp: Found irqno %d " 5019 "from cache for device %s, instance #%d\n", *pci_irqp, 5020 ddi_get_name(dip), ddi_get_instance(dip))); 5021 return (status); 5022 } 5023 5024 bzero(&acpipsmlnk, sizeof (acpi_psm_lnk_t)); 5025 5026 if ((status = acpi_translate_pci_irq(dip, ipin, pci_irqp, intr_flagp, 5027 &acpipsmlnk)) == ACPI_PSM_FAILURE) { 5028 APIC_VERBOSE_IRQ((CE_WARN, "pcplusmp: " 5029 " acpi_translate_pci_irq failed for device %s, instance" 5030 " #%d", ddi_get_name(dip), ddi_get_instance(dip))); 5031 return (status); 5032 } 5033 5034 if (status == ACPI_PSM_PARTIAL && acpipsmlnk.lnkobj != NULL) { 5035 status = apic_acpi_irq_configure(&acpipsmlnk, dip, pci_irqp, 5036 intr_flagp); 5037 if (status != ACPI_PSM_SUCCESS) { 5038 status = acpi_get_current_irq_resource(&acpipsmlnk, 5039 pci_irqp, intr_flagp); 5040 } 5041 } 5042 5043 if (status == ACPI_PSM_SUCCESS) { 5044 acpi_new_irq_cache_ent(busid, devid, ipin, *pci_irqp, 5045 intr_flagp, &acpipsmlnk); 5046 5047 APIC_VERBOSE_IRQ((CE_CONT, "pcplusmp: [ACPI] " 5048 "new irq %d for device %s, instance #%d\n", 5049 *pci_irqp, ddi_get_name(dip), ddi_get_instance(dip))); 5050 } 5051 5052 return (status); 5053 } 5054 5055 /* 5056 * Configures the irq for the interrupt link device identified by 5057 * acpipsmlnkp. 5058 * 5059 * Gets the current and the list of possible irq settings for the 5060 * device. If apic_unconditional_srs is not set, and the current 5061 * resource setting is in the list of possible irq settings, 5062 * current irq resource setting is passed to the caller. 5063 * 5064 * Otherwise, picks an irq number from the list of possible irq 5065 * settings, and sets the irq of the device to this value. 5066 * If prefer_crs is set, among a set of irq numbers in the list that have 5067 * the least number of devices sharing the interrupt, we pick current irq 5068 * resource setting if it is a member of this set. 5069 * 5070 * Passes the irq number in the value pointed to by pci_irqp, and 5071 * polarity and sensitivity in the structure pointed to by dipintrflagp 5072 * to the caller. 5073 * 5074 * Note that if setting the irq resource failed, but successfuly obtained 5075 * the current irq resource settings, passes the current irq resources 5076 * and considers it a success. 5077 * 5078 * Returns: 5079 * ACPI_PSM_SUCCESS on success. 5080 * 5081 * ACPI_PSM_FAILURE if an error occured during the configuration or 5082 * if a suitable irq was not found for this device, or if setting the 5083 * irq resource and obtaining the current resource fails. 5084 * 5085 */ 5086 static int 5087 apic_acpi_irq_configure(acpi_psm_lnk_t *acpipsmlnkp, dev_info_t *dip, 5088 int *pci_irqp, iflag_t *dipintr_flagp) 5089 { 5090 5091 int i, min_share, foundnow, done = 0; 5092 int32_t irq; 5093 int32_t share_irq = -1; 5094 int32_t chosen_irq = -1; 5095 int cur_irq = -1; 5096 acpi_irqlist_t *irqlistp; 5097 acpi_irqlist_t *irqlistent; 5098 5099 if ((acpi_get_possible_irq_resources(acpipsmlnkp, &irqlistp)) 5100 == ACPI_PSM_FAILURE) { 5101 APIC_VERBOSE_IRQ((CE_WARN, "!pcplusmp: Unable to determine " 5102 "or assign IRQ for device %s, instance #%d: The system was " 5103 "unable to get the list of potential IRQs from ACPI.", 5104 ddi_get_name(dip), ddi_get_instance(dip))); 5105 5106 return (ACPI_PSM_FAILURE); 5107 } 5108 5109 if ((acpi_get_current_irq_resource(acpipsmlnkp, &cur_irq, 5110 dipintr_flagp) == ACPI_PSM_SUCCESS) && (!apic_unconditional_srs) && 5111 (cur_irq > 0)) { 5112 /* 5113 * If an IRQ is set in CRS and that IRQ exists in the set 5114 * returned from _PRS, return that IRQ, otherwise print 5115 * a warning 5116 */ 5117 5118 if (acpi_irqlist_find_irq(irqlistp, cur_irq, NULL) 5119 == ACPI_PSM_SUCCESS) { 5120 5121 acpi_free_irqlist(irqlistp); 5122 ASSERT(pci_irqp != NULL); 5123 *pci_irqp = cur_irq; 5124 return (ACPI_PSM_SUCCESS); 5125 } 5126 5127 APIC_VERBOSE_IRQ((CE_WARN, "!pcplusmp: Could not find the " 5128 "current irq %d for device %s, instance #%d in ACPI's " 5129 "list of possible irqs for this device. Picking one from " 5130 " the latter list.", cur_irq, ddi_get_name(dip), 5131 ddi_get_instance(dip))); 5132 } 5133 5134 irqlistent = irqlistp; 5135 min_share = 255; 5136 5137 while (irqlistent != NULL) { 5138 irqlistent->intr_flags.bustype = BUS_PCI; 5139 5140 for (foundnow = 0, i = 0; i < irqlistent->num_irqs; i++) { 5141 5142 irq = irqlistent->irqs[i]; 5143 5144 if ((irq < 16) && (apic_reserved_irqlist[irq])) 5145 continue; 5146 5147 if (irq == 0) { 5148 /* invalid irq number */ 5149 continue; 5150 } 5151 5152 if ((apic_irq_table[irq] == NULL) || 5153 (apic_irq_table[irq]->airq_dip == dip)) { 5154 chosen_irq = irq; 5155 foundnow = 1; 5156 /* 5157 * If we do not prefer current irq from crs 5158 * or if we do and this irq is the same as 5159 * current irq from crs, this is the one 5160 * to pick. 5161 */ 5162 if (!(apic_prefer_crs) || (irq == cur_irq)) { 5163 done = 1; 5164 break; 5165 } 5166 continue; 5167 } 5168 5169 if (irqlistent->intr_flags.intr_el == INTR_EL_EDGE) 5170 continue; 5171 5172 if (!acpi_intr_compatible(irqlistent->intr_flags, 5173 apic_irq_table[irq]->airq_iflag)) 5174 continue; 5175 5176 if ((apic_irq_table[irq]->airq_share < min_share) || 5177 ((apic_irq_table[irq]->airq_share == min_share) && 5178 (cur_irq == irq) && (apic_prefer_crs))) { 5179 min_share = apic_irq_table[irq]->airq_share; 5180 share_irq = irq; 5181 foundnow = 1; 5182 } 5183 } 5184 5185 /* 5186 * If we found an IRQ in the inner loop this time, save the 5187 * details from the irqlist for later use. 5188 */ 5189 if (foundnow && ((chosen_irq != -1) || (share_irq != -1))) { 5190 /* 5191 * Copy the acpi_prs_private_t and flags from this 5192 * irq list entry, since we found an irq from this 5193 * entry. 5194 */ 5195 acpipsmlnkp->acpi_prs_prv = irqlistent->acpi_prs_prv; 5196 *dipintr_flagp = irqlistent->intr_flags; 5197 } 5198 5199 if (done) 5200 break; 5201 5202 /* Go to the next irqlist entry */ 5203 irqlistent = irqlistent->next; 5204 } 5205 5206 5207 acpi_free_irqlist(irqlistp); 5208 if (chosen_irq != -1) 5209 irq = chosen_irq; 5210 else if (share_irq != -1) 5211 irq = share_irq; 5212 else { 5213 APIC_VERBOSE_IRQ((CE_WARN, "!pcplusmp: Could not find a " 5214 "suitable irq from the list of possible irqs for device " 5215 "%s, instance #%d in ACPI's list of possible irqs", 5216 ddi_get_name(dip), ddi_get_instance(dip))); 5217 return (ACPI_PSM_FAILURE); 5218 } 5219 5220 APIC_VERBOSE_IRQ((CE_CONT, "!pcplusmp: Setting irq %d for device %s " 5221 "instance #%d\n", irq, ddi_get_name(dip), ddi_get_instance(dip))); 5222 5223 if ((acpi_set_irq_resource(acpipsmlnkp, irq)) == ACPI_PSM_SUCCESS) { 5224 /* 5225 * setting irq was successful, check to make sure CRS 5226 * reflects that. If CRS does not agree with what we 5227 * set, return the irq that was set. 5228 */ 5229 5230 if (acpi_get_current_irq_resource(acpipsmlnkp, &cur_irq, 5231 dipintr_flagp) == ACPI_PSM_SUCCESS) { 5232 5233 if (cur_irq != irq) 5234 APIC_VERBOSE_IRQ((CE_WARN, "!pcplusmp: " 5235 "IRQ resource set (irqno %d) for device %s " 5236 "instance #%d, differs from current " 5237 "setting irqno %d", 5238 irq, ddi_get_name(dip), 5239 ddi_get_instance(dip), cur_irq)); 5240 } 5241 5242 /* 5243 * return the irq that was set, and not what CRS reports, 5244 * since CRS has been seen to be bogus on some systems 5245 */ 5246 cur_irq = irq; 5247 } else { 5248 APIC_VERBOSE_IRQ((CE_WARN, "!pcplusmp: set resource irq %d " 5249 "failed for device %s instance #%d", 5250 irq, ddi_get_name(dip), ddi_get_instance(dip))); 5251 5252 if (cur_irq == -1) 5253 return (ACPI_PSM_FAILURE); 5254 } 5255 5256 ASSERT(pci_irqp != NULL); 5257 *pci_irqp = cur_irq; 5258 return (ACPI_PSM_SUCCESS); 5259 } 5260