1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2017 Joyent, Inc. 24 */ 25 /* 26 * Copyright (c) 2010, Intel Corporation. 27 * All rights reserved. 28 */ 29 30 /* 31 * PSMI 1.1 extensions are supported only in 2.6 and later versions. 32 * PSMI 1.2 extensions are supported only in 2.7 and later versions. 33 * PSMI 1.3 and 1.4 extensions are supported in Solaris 10. 34 * PSMI 1.5 extensions are supported in Solaris Nevada. 35 * PSMI 1.6 extensions are supported in Solaris Nevada. 36 * PSMI 1.7 extensions are supported in Solaris Nevada. 37 */ 38 #define PSMI_1_7 39 40 #include <sys/processor.h> 41 #include <sys/time.h> 42 #include <sys/psm.h> 43 #include <sys/smp_impldefs.h> 44 #include <sys/inttypes.h> 45 #include <sys/cram.h> 46 #include <sys/acpi/acpi.h> 47 #include <sys/acpica.h> 48 #include <sys/psm_common.h> 49 #include <sys/apic.h> 50 #include <sys/apic_common.h> 51 #include <sys/pit.h> 52 #include <sys/ddi.h> 53 #include <sys/sunddi.h> 54 #include <sys/ddi_impldefs.h> 55 #include <sys/pci.h> 56 #include <sys/promif.h> 57 #include <sys/x86_archext.h> 58 #include <sys/cpc_impl.h> 59 #include <sys/uadmin.h> 60 #include <sys/panic.h> 61 #include <sys/debug.h> 62 #include <sys/archsystm.h> 63 #include <sys/trap.h> 64 #include <sys/machsystm.h> 65 #include <sys/cpuvar.h> 66 #include <sys/rm_platter.h> 67 #include <sys/privregs.h> 68 #include <sys/cyclic.h> 69 #include <sys/note.h> 70 #include <sys/pci_intr_lib.h> 71 #include <sys/sunndi.h> 72 #include <sys/hpet.h> 73 #include <sys/clock.h> 74 75 /* 76 * Part of mp_platfrom_common.c that's used only by pcplusmp & xpv_psm 77 * but not apix. 78 * These functions may be moved to xpv_psm later when apix and pcplusmp 79 * are merged together 80 */ 81 82 /* 83 * Local Function Prototypes 84 */ 85 static void apic_mark_vector(uchar_t oldvector, uchar_t newvector); 86 static void apic_xlate_vector_free_timeout_handler(void *arg); 87 static int apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu, 88 int new_bind_cpu, int apicindex, int intin_no, int which_irq, 89 struct ioapic_reprogram_data *drep); 90 static int apic_setup_irq_table(dev_info_t *dip, int irqno, 91 struct apic_io_intr *intrp, struct intrspec *ispec, iflag_t *intr_flagp, 92 int type); 93 static void apic_try_deferred_reprogram(int ipl, int vect); 94 static void delete_defer_repro_ent(int which_irq); 95 static void apic_ioapic_wait_pending_clear(int ioapicindex, 96 int intin_no); 97 98 extern int apic_acpi_translate_pci_irq(dev_info_t *dip, int busid, int devid, 99 int ipin, int *pci_irqp, iflag_t *intr_flagp); 100 extern int apic_handle_pci_pci_bridge(dev_info_t *idip, int child_devno, 101 int child_ipin, struct apic_io_intr **intrp); 102 extern uchar_t acpi_find_ioapic(int irq); 103 extern struct apic_io_intr *apic_find_io_intr_w_busid(int irqno, int busid); 104 extern int apic_find_bus_id(int bustype); 105 extern int apic_find_intin(uchar_t ioapic, uchar_t intin); 106 extern void apic_record_rdt_entry(apic_irq_t *irqptr, int irq); 107 108 extern int apic_sci_vect; 109 extern iflag_t apic_sci_flags; 110 /* ACPI HPET interrupt configuration; -1 if HPET not used */ 111 extern int apic_hpet_vect; 112 extern iflag_t apic_hpet_flags; 113 extern int apic_intr_policy; 114 extern char *psm_name; 115 116 /* 117 * number of bits per byte, from <sys/param.h> 118 */ 119 #define UCHAR_MAX UINT8_MAX 120 121 /* Max wait time (in repetitions) for flags to clear in an RDT entry. */ 122 extern int apic_max_reps_clear_pending; 123 124 /* The irq # is implicit in the array index: */ 125 struct ioapic_reprogram_data apic_reprogram_info[APIC_MAX_VECTOR+1]; 126 /* 127 * APIC_MAX_VECTOR + 1 is the maximum # of IRQs as well. ioapic_reprogram_info 128 * is indexed by IRQ number, NOT by vector number. 129 */ 130 131 extern int apic_int_busy_mark; 132 extern int apic_int_free_mark; 133 extern int apic_diff_for_redistribution; 134 extern int apic_sample_factor_redistribution; 135 extern int apic_redist_cpu_skip; 136 extern int apic_num_imbalance; 137 extern int apic_num_rebind; 138 139 /* timeout for xlate_vector, mark_vector */ 140 int apic_revector_timeout = 16 * 10000; /* 160 millisec */ 141 142 extern int apic_defconf; 143 extern int apic_irq_translate; 144 145 extern int apic_use_acpi_madt_only; /* 1=ONLY use MADT from ACPI */ 146 147 extern uchar_t apic_io_vectbase[MAX_IO_APIC]; 148 149 extern boolean_t ioapic_mask_workaround[MAX_IO_APIC]; 150 151 /* 152 * First available slot to be used as IRQ index into the apic_irq_table 153 * for those interrupts (like MSI/X) that don't have a physical IRQ. 154 */ 155 extern int apic_first_avail_irq; 156 157 /* 158 * apic_defer_reprogram_lock ensures that only one processor is handling 159 * deferred interrupt programming at *_intr_exit time. 160 */ 161 static lock_t apic_defer_reprogram_lock; 162 163 /* 164 * The current number of deferred reprogrammings outstanding 165 */ 166 uint_t apic_reprogram_outstanding = 0; 167 168 #ifdef DEBUG 169 /* 170 * Counters that keep track of deferred reprogramming stats 171 */ 172 uint_t apic_intr_deferrals = 0; 173 uint_t apic_intr_deliver_timeouts = 0; 174 uint_t apic_last_ditch_reprogram_failures = 0; 175 uint_t apic_deferred_setup_failures = 0; 176 uint_t apic_defer_repro_total_retries = 0; 177 uint_t apic_defer_repro_successes = 0; 178 uint_t apic_deferred_spurious_enters = 0; 179 #endif 180 181 extern int apic_io_max; 182 extern struct apic_io_intr *apic_io_intrp; 183 184 uchar_t apic_vector_to_irq[APIC_MAX_VECTOR+1]; 185 186 extern uint32_t eisa_level_intr_mask; 187 /* At least MSB will be set if EISA bus */ 188 189 extern int apic_pci_bus_total; 190 extern uchar_t apic_single_pci_busid; 191 192 /* 193 * Following declarations are for revectoring; used when ISRs at different 194 * IPLs share an irq. 195 */ 196 static lock_t apic_revector_lock; 197 int apic_revector_pending = 0; 198 static uchar_t *apic_oldvec_to_newvec; 199 static uchar_t *apic_newvec_to_oldvec; 200 201 /* ACPI Interrupt Source Override Structure ptr */ 202 extern ACPI_MADT_INTERRUPT_OVERRIDE *acpi_isop; 203 extern int acpi_iso_cnt; 204 205 /* 206 * Auto-configuration routines 207 */ 208 209 /* 210 * Initialise vector->ipl and ipl->pri arrays. level_intr and irqtable 211 * are also set to NULL. vector->irq is set to a value which cannot map 212 * to a real irq to show that it is free. 213 */ 214 void 215 apic_init_common(void) 216 { 217 int i, j, indx; 218 int *iptr; 219 220 /* 221 * Initialize apic_ipls from apic_vectortoipl. This array is 222 * used in apic_intr_enter to determine the IPL to use for the 223 * corresponding vector. On some systems, due to hardware errata 224 * and interrupt sharing, the IPL may not correspond to the IPL listed 225 * in apic_vectortoipl (see apic_addspl and apic_delspl). 226 */ 227 for (i = 0; i < (APIC_AVAIL_VECTOR / APIC_VECTOR_PER_IPL); i++) { 228 indx = i * APIC_VECTOR_PER_IPL; 229 230 for (j = 0; j < APIC_VECTOR_PER_IPL; j++, indx++) 231 apic_ipls[indx] = apic_vectortoipl[i]; 232 } 233 234 /* cpu 0 is always up (for now) */ 235 apic_cpus[0].aci_status = APIC_CPU_ONLINE | APIC_CPU_INTR_ENABLE; 236 237 iptr = (int *)&apic_irq_table[0]; 238 for (i = 0; i <= APIC_MAX_VECTOR; i++) { 239 apic_level_intr[i] = 0; 240 *iptr++ = NULL; 241 apic_vector_to_irq[i] = APIC_RESV_IRQ; 242 243 /* These *must* be initted to B_TRUE! */ 244 apic_reprogram_info[i].done = B_TRUE; 245 apic_reprogram_info[i].irqp = NULL; 246 apic_reprogram_info[i].tries = 0; 247 apic_reprogram_info[i].bindcpu = 0; 248 } 249 250 /* 251 * Allocate a dummy irq table entry for the reserved entry. 252 * This takes care of the race between removing an irq and 253 * clock detecting a CPU in that irq during interrupt load 254 * sampling. 255 */ 256 apic_irq_table[APIC_RESV_IRQ] = 257 kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP); 258 259 mutex_init(&airq_mutex, NULL, MUTEX_DEFAULT, NULL); 260 } 261 262 void 263 ioapic_init_intr(int mask_apic) 264 { 265 int ioapic_ix; 266 struct intrspec ispec; 267 apic_irq_t *irqptr; 268 int i, j; 269 ulong_t iflag; 270 271 LOCK_INIT_CLEAR(&apic_revector_lock); 272 LOCK_INIT_CLEAR(&apic_defer_reprogram_lock); 273 274 /* mask interrupt vectors */ 275 for (j = 0; j < apic_io_max && mask_apic; j++) { 276 int intin_max; 277 278 ioapic_ix = j; 279 /* Bits 23-16 define the maximum redirection entries */ 280 intin_max = (ioapic_read(ioapic_ix, APIC_VERS_CMD) >> 16) 281 & 0xff; 282 for (i = 0; i <= intin_max; i++) 283 ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * i, AV_MASK); 284 } 285 286 /* 287 * Hack alert: deal with ACPI SCI interrupt chicken/egg here 288 */ 289 if (apic_sci_vect > 0) { 290 /* 291 * acpica has already done add_avintr(); we just 292 * to finish the job by mimicing translate_irq() 293 * 294 * Fake up an intrspec and setup the tables 295 */ 296 ispec.intrspec_vec = apic_sci_vect; 297 ispec.intrspec_pri = SCI_IPL; 298 299 if (apic_setup_irq_table(NULL, apic_sci_vect, NULL, 300 &ispec, &apic_sci_flags, DDI_INTR_TYPE_FIXED) < 0) { 301 cmn_err(CE_WARN, "!apic: SCI setup failed"); 302 return; 303 } 304 irqptr = apic_irq_table[apic_sci_vect]; 305 306 iflag = intr_clear(); 307 lock_set(&apic_ioapic_lock); 308 309 /* Program I/O APIC */ 310 (void) apic_setup_io_intr(irqptr, apic_sci_vect, B_FALSE); 311 312 lock_clear(&apic_ioapic_lock); 313 intr_restore(iflag); 314 315 irqptr->airq_share++; 316 } 317 318 /* 319 * Hack alert: deal with ACPI HPET interrupt chicken/egg here. 320 */ 321 if (apic_hpet_vect > 0) { 322 /* 323 * hpet has already done add_avintr(); we just need 324 * to finish the job by mimicing translate_irq() 325 * 326 * Fake up an intrspec and setup the tables 327 */ 328 ispec.intrspec_vec = apic_hpet_vect; 329 ispec.intrspec_pri = CBE_HIGH_PIL; 330 331 if (apic_setup_irq_table(NULL, apic_hpet_vect, NULL, 332 &ispec, &apic_hpet_flags, DDI_INTR_TYPE_FIXED) < 0) { 333 cmn_err(CE_WARN, "!apic: HPET setup failed"); 334 return; 335 } 336 irqptr = apic_irq_table[apic_hpet_vect]; 337 338 iflag = intr_clear(); 339 lock_set(&apic_ioapic_lock); 340 341 /* Program I/O APIC */ 342 (void) apic_setup_io_intr(irqptr, apic_hpet_vect, B_FALSE); 343 344 lock_clear(&apic_ioapic_lock); 345 intr_restore(iflag); 346 347 irqptr->airq_share++; 348 } 349 } 350 351 /* 352 * Add mask bits to disable interrupt vector from happening 353 * at or above IPL. In addition, it should remove mask bits 354 * to enable interrupt vectors below the given IPL. 355 * 356 * Both add and delspl are complicated by the fact that different interrupts 357 * may share IRQs. This can happen in two ways. 358 * 1. The same H/W line is shared by more than 1 device 359 * 1a. with interrupts at different IPLs 360 * 1b. with interrupts at same IPL 361 * 2. We ran out of vectors at a given IPL and started sharing vectors. 362 * 1b and 2 should be handled gracefully, except for the fact some ISRs 363 * will get called often when no interrupt is pending for the device. 364 * For 1a, we handle it at the higher IPL. 365 */ 366 /*ARGSUSED*/ 367 int 368 apic_addspl_common(int irqno, int ipl, int min_ipl, int max_ipl) 369 { 370 uchar_t vector; 371 ulong_t iflag; 372 apic_irq_t *irqptr, *irqheadptr; 373 int irqindex; 374 375 ASSERT(max_ipl <= UCHAR_MAX); 376 irqindex = IRQINDEX(irqno); 377 378 if ((irqindex == -1) || (!apic_irq_table[irqindex])) 379 return (PSM_FAILURE); 380 381 mutex_enter(&airq_mutex); 382 irqptr = irqheadptr = apic_irq_table[irqindex]; 383 384 DDI_INTR_IMPLDBG((CE_CONT, "apic_addspl: dip=0x%p type=%d irqno=0x%x " 385 "vector=0x%x\n", (void *)irqptr->airq_dip, 386 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector)); 387 388 while (irqptr) { 389 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno) 390 break; 391 irqptr = irqptr->airq_next; 392 } 393 irqptr->airq_share++; 394 395 mutex_exit(&airq_mutex); 396 397 /* return if it is not hardware interrupt */ 398 if (irqptr->airq_mps_intr_index == RESERVE_INDEX) 399 return (PSM_SUCCESS); 400 401 /* Or if there are more interupts at a higher IPL */ 402 if (ipl != max_ipl) 403 return (PSM_SUCCESS); 404 405 /* 406 * if apic_picinit() has not been called yet, just return. 407 * At the end of apic_picinit(), we will call setup_io_intr(). 408 */ 409 410 if (!apic_picinit_called) 411 return (PSM_SUCCESS); 412 413 /* 414 * Upgrade vector if max_ipl is not earlier ipl. If we cannot allocate, 415 * return failure. 416 */ 417 if (irqptr->airq_ipl != max_ipl && 418 !ioapic_mask_workaround[irqptr->airq_ioapicindex]) { 419 420 vector = apic_allocate_vector(max_ipl, irqindex, 1); 421 if (vector == 0) { 422 irqptr->airq_share--; 423 return (PSM_FAILURE); 424 } 425 irqptr = irqheadptr; 426 apic_mark_vector(irqptr->airq_vector, vector); 427 while (irqptr) { 428 irqptr->airq_vector = vector; 429 irqptr->airq_ipl = (uchar_t)max_ipl; 430 /* 431 * reprogram irq being added and every one else 432 * who is not in the UNINIT state 433 */ 434 if ((VIRTIRQ(irqindex, irqptr->airq_share_id) == 435 irqno) || (irqptr->airq_temp_cpu != IRQ_UNINIT)) { 436 apic_record_rdt_entry(irqptr, irqindex); 437 438 iflag = intr_clear(); 439 lock_set(&apic_ioapic_lock); 440 441 (void) apic_setup_io_intr(irqptr, irqindex, 442 B_FALSE); 443 444 lock_clear(&apic_ioapic_lock); 445 intr_restore(iflag); 446 } 447 irqptr = irqptr->airq_next; 448 } 449 return (PSM_SUCCESS); 450 451 } else if (irqptr->airq_ipl != max_ipl && 452 ioapic_mask_workaround[irqptr->airq_ioapicindex]) { 453 /* 454 * We cannot upgrade the vector, but we can change 455 * the IPL that this vector induces. 456 * 457 * Note that we subtract APIC_BASE_VECT from the vector 458 * here because this array is used in apic_intr_enter 459 * (no need to add APIC_BASE_VECT in that hot code 460 * path since we can do it in the rarely-executed path 461 * here). 462 */ 463 apic_ipls[irqptr->airq_vector - APIC_BASE_VECT] = 464 (uchar_t)max_ipl; 465 466 irqptr = irqheadptr; 467 while (irqptr) { 468 irqptr->airq_ipl = (uchar_t)max_ipl; 469 irqptr = irqptr->airq_next; 470 } 471 472 return (PSM_SUCCESS); 473 } 474 475 ASSERT(irqptr); 476 477 iflag = intr_clear(); 478 lock_set(&apic_ioapic_lock); 479 480 (void) apic_setup_io_intr(irqptr, irqindex, B_FALSE); 481 482 lock_clear(&apic_ioapic_lock); 483 intr_restore(iflag); 484 485 return (PSM_SUCCESS); 486 } 487 488 /* 489 * Recompute mask bits for the given interrupt vector. 490 * If there is no interrupt servicing routine for this 491 * vector, this function should disable interrupt vector 492 * from happening at all IPLs. If there are still 493 * handlers using the given vector, this function should 494 * disable the given vector from happening below the lowest 495 * IPL of the remaining hadlers. 496 */ 497 /*ARGSUSED*/ 498 int 499 apic_delspl_common(int irqno, int ipl, int min_ipl, int max_ipl) 500 { 501 uchar_t vector; 502 uint32_t bind_cpu; 503 int intin, irqindex; 504 int ioapic_ix; 505 apic_irq_t *irqptr, *preirqptr, *irqheadptr, *irqp; 506 ulong_t iflag; 507 508 mutex_enter(&airq_mutex); 509 irqindex = IRQINDEX(irqno); 510 irqptr = preirqptr = irqheadptr = apic_irq_table[irqindex]; 511 512 DDI_INTR_IMPLDBG((CE_CONT, "apic_delspl: dip=0x%p type=%d irqno=0x%x " 513 "vector=0x%x\n", (void *)irqptr->airq_dip, 514 irqptr->airq_mps_intr_index, irqno, irqptr->airq_vector)); 515 516 while (irqptr) { 517 if (VIRTIRQ(irqindex, irqptr->airq_share_id) == irqno) 518 break; 519 preirqptr = irqptr; 520 irqptr = irqptr->airq_next; 521 } 522 ASSERT(irqptr); 523 524 irqptr->airq_share--; 525 526 mutex_exit(&airq_mutex); 527 528 /* 529 * If there are more interrupts at a higher IPL, we don't need 530 * to disable anything. 531 */ 532 if (ipl < max_ipl) 533 return (PSM_SUCCESS); 534 535 /* return if it is not hardware interrupt */ 536 if (irqptr->airq_mps_intr_index == RESERVE_INDEX) 537 return (PSM_SUCCESS); 538 539 if (!apic_picinit_called) { 540 /* 541 * Clear irq_struct. If two devices shared an intpt 542 * line & 1 unloaded before picinit, we are hosed. But, then 543 * we hope the machine survive. 544 */ 545 irqptr->airq_mps_intr_index = FREE_INDEX; 546 irqptr->airq_temp_cpu = IRQ_UNINIT; 547 apic_free_vector(irqptr->airq_vector); 548 return (PSM_SUCCESS); 549 } 550 /* 551 * Downgrade vector to new max_ipl if needed. If we cannot allocate, 552 * use old IPL. Not very elegant, but it should work. 553 */ 554 if ((irqptr->airq_ipl != max_ipl) && (max_ipl != PSM_INVALID_IPL) && 555 !ioapic_mask_workaround[irqptr->airq_ioapicindex]) { 556 apic_irq_t *irqp; 557 if ((vector = apic_allocate_vector(max_ipl, irqno, 1))) { 558 apic_mark_vector(irqheadptr->airq_vector, vector); 559 irqp = irqheadptr; 560 while (irqp) { 561 irqp->airq_vector = vector; 562 irqp->airq_ipl = (uchar_t)max_ipl; 563 if (irqp->airq_temp_cpu != IRQ_UNINIT) { 564 apic_record_rdt_entry(irqp, irqindex); 565 566 iflag = intr_clear(); 567 lock_set(&apic_ioapic_lock); 568 569 (void) apic_setup_io_intr(irqp, 570 irqindex, B_FALSE); 571 572 lock_clear(&apic_ioapic_lock); 573 intr_restore(iflag); 574 } 575 irqp = irqp->airq_next; 576 } 577 } 578 579 } else if (irqptr->airq_ipl != max_ipl && 580 max_ipl != PSM_INVALID_IPL && 581 ioapic_mask_workaround[irqptr->airq_ioapicindex]) { 582 583 /* 584 * We cannot downgrade the IPL of the vector below the vector's 585 * hardware priority. If we did, it would be possible for a 586 * higher-priority hardware vector to interrupt a CPU running at an IPL 587 * lower than the hardware priority of the interrupting vector (but 588 * higher than the soft IPL of this IRQ). When this happens, we would 589 * then try to drop the IPL BELOW what it was (effectively dropping 590 * below base_spl) which would be potentially catastrophic. 591 * 592 * (e.g. Suppose the hardware vector associated with this IRQ is 0x40 593 * (hardware IPL of 4). Further assume that the old IPL of this IRQ 594 * was 4, but the new IPL is 1. If we forced vector 0x40 to result in 595 * an IPL of 1, it would be possible for the processor to be executing 596 * at IPL 3 and for an interrupt to come in on vector 0x40, interrupting 597 * the currently-executing ISR. When apic_intr_enter consults 598 * apic_irqs[], it will return 1, bringing the IPL of the CPU down to 1 599 * so even though the processor was running at IPL 4, an IPL 1 600 * interrupt will have interrupted it, which must not happen)). 601 * 602 * Effectively, this means that the hardware priority corresponding to 603 * the IRQ's IPL (in apic_ipls[]) cannot be lower than the vector's 604 * hardware priority. 605 * 606 * (In the above example, then, after removal of the IPL 4 device's 607 * interrupt handler, the new IPL will continue to be 4 because the 608 * hardware priority that IPL 1 implies is lower than the hardware 609 * priority of the vector used.) 610 */ 611 /* apic_ipls is indexed by vector, starting at APIC_BASE_VECT */ 612 const int apic_ipls_index = irqptr->airq_vector - 613 APIC_BASE_VECT; 614 const int vect_inherent_hwpri = irqptr->airq_vector >> 615 APIC_IPL_SHIFT; 616 617 /* 618 * If there are still devices using this IRQ, determine the 619 * new ipl to use. 620 */ 621 if (irqptr->airq_share) { 622 int vect_desired_hwpri, hwpri; 623 624 ASSERT(max_ipl < MAXIPL); 625 vect_desired_hwpri = apic_ipltopri[max_ipl] >> 626 APIC_IPL_SHIFT; 627 628 /* 629 * If the desired IPL's hardware priority is lower 630 * than that of the vector, use the hardware priority 631 * of the vector to determine the new IPL. 632 */ 633 hwpri = (vect_desired_hwpri < vect_inherent_hwpri) ? 634 vect_inherent_hwpri : vect_desired_hwpri; 635 636 /* 637 * Now, to get the right index for apic_vectortoipl, 638 * we need to subtract APIC_BASE_VECT from the 639 * hardware-vector-equivalent (in hwpri). Since hwpri 640 * is already shifted, we shift APIC_BASE_VECT before 641 * doing the subtraction. 642 */ 643 hwpri -= (APIC_BASE_VECT >> APIC_IPL_SHIFT); 644 645 ASSERT(hwpri >= 0); 646 ASSERT(hwpri < MAXIPL); 647 max_ipl = apic_vectortoipl[hwpri]; 648 apic_ipls[apic_ipls_index] = (uchar_t)max_ipl; 649 650 irqp = irqheadptr; 651 while (irqp) { 652 irqp->airq_ipl = (uchar_t)max_ipl; 653 irqp = irqp->airq_next; 654 } 655 } else { 656 /* 657 * No more devices on this IRQ, so reset this vector's 658 * element in apic_ipls to the original IPL for this 659 * vector 660 */ 661 apic_ipls[apic_ipls_index] = 662 apic_vectortoipl[vect_inherent_hwpri]; 663 } 664 } 665 666 /* 667 * If there are still active interrupts, we are done. 668 */ 669 if (irqptr->airq_share) 670 return (PSM_SUCCESS); 671 672 iflag = intr_clear(); 673 lock_set(&apic_ioapic_lock); 674 675 if (irqptr->airq_mps_intr_index == MSI_INDEX) { 676 /* 677 * Disable the MSI vector 678 * Make sure we only disable on the last 679 * of the multi-MSI support 680 */ 681 if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) { 682 apic_pci_msi_disable_mode(irqptr->airq_dip, 683 DDI_INTR_TYPE_MSI); 684 } 685 } else if (irqptr->airq_mps_intr_index == MSIX_INDEX) { 686 /* 687 * Disable the MSI-X vector 688 * needs to clear its mask and addr/data for each MSI-X 689 */ 690 apic_pci_msi_unconfigure(irqptr->airq_dip, DDI_INTR_TYPE_MSIX, 691 irqptr->airq_origirq); 692 /* 693 * Make sure we only disable on the last MSI-X 694 */ 695 if (i_ddi_intr_get_current_nenables(irqptr->airq_dip) == 1) { 696 apic_pci_msi_disable_mode(irqptr->airq_dip, 697 DDI_INTR_TYPE_MSIX); 698 } 699 } else { 700 /* 701 * The assumption here is that this is safe, even for 702 * systems with IOAPICs that suffer from the hardware 703 * erratum because all devices have been quiesced before 704 * they unregister their interrupt handlers. If that 705 * assumption turns out to be false, this mask operation 706 * can induce the same erratum result we're trying to 707 * avoid. 708 */ 709 ioapic_ix = irqptr->airq_ioapicindex; 710 intin = irqptr->airq_intin_no; 711 ioapic_write(ioapic_ix, APIC_RDT_CMD + 2 * intin, AV_MASK); 712 } 713 714 apic_vt_ops->apic_intrmap_free_entry(&irqptr->airq_intrmap_private); 715 716 /* 717 * This irq entry is the only one in the chain. 718 */ 719 if (irqheadptr->airq_next == NULL) { 720 ASSERT(irqheadptr == irqptr); 721 bind_cpu = irqptr->airq_temp_cpu; 722 if (((uint32_t)bind_cpu != IRQ_UNBOUND) && 723 ((uint32_t)bind_cpu != IRQ_UNINIT)) { 724 ASSERT(apic_cpu_in_range(bind_cpu)); 725 if (bind_cpu & IRQ_USER_BOUND) { 726 /* If hardbound, temp_cpu == cpu */ 727 bind_cpu &= ~IRQ_USER_BOUND; 728 apic_cpus[bind_cpu].aci_bound--; 729 } else 730 apic_cpus[bind_cpu].aci_temp_bound--; 731 } 732 irqptr->airq_temp_cpu = IRQ_UNINIT; 733 irqptr->airq_mps_intr_index = FREE_INDEX; 734 lock_clear(&apic_ioapic_lock); 735 intr_restore(iflag); 736 apic_free_vector(irqptr->airq_vector); 737 return (PSM_SUCCESS); 738 } 739 740 /* 741 * If we get here, we are sharing the vector and there are more than 742 * one active irq entries in the chain. 743 */ 744 lock_clear(&apic_ioapic_lock); 745 intr_restore(iflag); 746 747 mutex_enter(&airq_mutex); 748 /* Remove the irq entry from the chain */ 749 if (irqptr == irqheadptr) { /* The irq entry is at the head */ 750 apic_irq_table[irqindex] = irqptr->airq_next; 751 } else { 752 preirqptr->airq_next = irqptr->airq_next; 753 } 754 /* Free the irq entry */ 755 kmem_free(irqptr, sizeof (apic_irq_t)); 756 mutex_exit(&airq_mutex); 757 758 return (PSM_SUCCESS); 759 } 760 761 /* 762 * apic_introp_xlate() replaces apic_translate_irq() and is 763 * called only from apic_intr_ops(). With the new ADII framework, 764 * the priority can no longer be retrieved through i_ddi_get_intrspec(). 765 * It has to be passed in from the caller. 766 * 767 * Return value: 768 * Success: irqno for the given device 769 * Failure: -1 770 */ 771 int 772 apic_introp_xlate(dev_info_t *dip, struct intrspec *ispec, int type) 773 { 774 char dev_type[16]; 775 int dev_len, pci_irq, newirq, bustype, devid, busid, i; 776 int irqno = ispec->intrspec_vec; 777 ddi_acc_handle_t cfg_handle; 778 uchar_t ipin; 779 struct apic_io_intr *intrp; 780 iflag_t intr_flag; 781 ACPI_SUBTABLE_HEADER *hp; 782 ACPI_MADT_INTERRUPT_OVERRIDE *isop; 783 apic_irq_t *airqp; 784 int parent_is_pci_or_pciex = 0; 785 int child_is_pciex = 0; 786 787 DDI_INTR_IMPLDBG((CE_CONT, "apic_introp_xlate: dip=0x%p name=%s " 788 "type=%d irqno=0x%x\n", (void *)dip, ddi_get_name(dip), type, 789 irqno)); 790 791 dev_len = sizeof (dev_type); 792 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, ddi_get_parent(dip), 793 DDI_PROP_DONTPASS, "device_type", (caddr_t)dev_type, 794 &dev_len) == DDI_PROP_SUCCESS) { 795 if ((strcmp(dev_type, "pci") == 0) || 796 (strcmp(dev_type, "pciex") == 0)) 797 parent_is_pci_or_pciex = 1; 798 } 799 800 if (ddi_getlongprop_buf(DDI_DEV_T_ANY, dip, 801 DDI_PROP_DONTPASS, "compatible", (caddr_t)dev_type, 802 &dev_len) == DDI_PROP_SUCCESS) { 803 if (strstr(dev_type, "pciex")) 804 child_is_pciex = 1; 805 } 806 807 if (DDI_INTR_IS_MSI_OR_MSIX(type)) { 808 if ((airqp = apic_find_irq(dip, ispec, type)) != NULL) { 809 airqp->airq_iflag.bustype = 810 child_is_pciex ? BUS_PCIE : BUS_PCI; 811 return (apic_vector_to_irq[airqp->airq_vector]); 812 } 813 return (apic_setup_irq_table(dip, irqno, NULL, ispec, 814 NULL, type)); 815 } 816 817 bustype = 0; 818 819 /* check if we have already translated this irq */ 820 mutex_enter(&airq_mutex); 821 newirq = apic_min_device_irq; 822 for (; newirq <= apic_max_device_irq; newirq++) { 823 airqp = apic_irq_table[newirq]; 824 while (airqp) { 825 if ((airqp->airq_dip == dip) && 826 (airqp->airq_origirq == irqno) && 827 (airqp->airq_mps_intr_index != FREE_INDEX)) { 828 829 mutex_exit(&airq_mutex); 830 return (VIRTIRQ(newirq, airqp->airq_share_id)); 831 } 832 airqp = airqp->airq_next; 833 } 834 } 835 mutex_exit(&airq_mutex); 836 837 if (apic_defconf) 838 goto defconf; 839 840 if ((dip == NULL) || (!apic_irq_translate && !apic_enable_acpi)) 841 goto nonpci; 842 843 if (parent_is_pci_or_pciex) { 844 /* pci device */ 845 if (acpica_get_bdf(dip, &busid, &devid, NULL) != 0) 846 goto nonpci; 847 if (busid == 0 && apic_pci_bus_total == 1) 848 busid = (int)apic_single_pci_busid; 849 850 if (pci_config_setup(dip, &cfg_handle) != DDI_SUCCESS) 851 return (-1); 852 ipin = pci_config_get8(cfg_handle, PCI_CONF_IPIN) - PCI_INTA; 853 pci_config_teardown(&cfg_handle); 854 if (apic_enable_acpi && !apic_use_acpi_madt_only) { 855 if (apic_acpi_translate_pci_irq(dip, busid, devid, 856 ipin, &pci_irq, &intr_flag) != ACPI_PSM_SUCCESS) 857 return (-1); 858 859 intr_flag.bustype = child_is_pciex ? BUS_PCIE : BUS_PCI; 860 return (apic_setup_irq_table(dip, pci_irq, NULL, ispec, 861 &intr_flag, type)); 862 } else { 863 pci_irq = ((devid & 0x1f) << 2) | (ipin & 0x3); 864 if ((intrp = apic_find_io_intr_w_busid(pci_irq, busid)) 865 == NULL) { 866 if ((pci_irq = apic_handle_pci_pci_bridge(dip, 867 devid, ipin, &intrp)) == -1) 868 return (-1); 869 } 870 return (apic_setup_irq_table(dip, pci_irq, intrp, ispec, 871 NULL, type)); 872 } 873 } else if (strcmp(dev_type, "isa") == 0) 874 bustype = BUS_ISA; 875 else if (strcmp(dev_type, "eisa") == 0) 876 bustype = BUS_EISA; 877 878 nonpci: 879 if (apic_enable_acpi && !apic_use_acpi_madt_only) { 880 /* search iso entries first */ 881 if (acpi_iso_cnt != 0) { 882 hp = (ACPI_SUBTABLE_HEADER *)acpi_isop; 883 i = 0; 884 while (i < acpi_iso_cnt) { 885 if (hp->Type == 886 ACPI_MADT_TYPE_INTERRUPT_OVERRIDE) { 887 isop = 888 (ACPI_MADT_INTERRUPT_OVERRIDE *) hp; 889 if (isop->Bus == 0 && 890 isop->SourceIrq == irqno) { 891 newirq = isop->GlobalIrq; 892 intr_flag.intr_po = 893 isop->IntiFlags & 894 ACPI_MADT_POLARITY_MASK; 895 intr_flag.intr_el = 896 (isop->IntiFlags & 897 ACPI_MADT_TRIGGER_MASK) 898 >> 2; 899 intr_flag.bustype = BUS_ISA; 900 901 return (apic_setup_irq_table( 902 dip, newirq, NULL, ispec, 903 &intr_flag, type)); 904 905 } 906 i++; 907 } 908 hp = (ACPI_SUBTABLE_HEADER *)(((char *)hp) + 909 hp->Length); 910 } 911 } 912 intr_flag.intr_po = INTR_PO_ACTIVE_HIGH; 913 intr_flag.intr_el = INTR_EL_EDGE; 914 intr_flag.bustype = BUS_ISA; 915 return (apic_setup_irq_table(dip, irqno, NULL, ispec, 916 &intr_flag, type)); 917 } else { 918 if (bustype == 0) /* not initialized */ 919 bustype = eisa_level_intr_mask ? BUS_EISA : BUS_ISA; 920 for (i = 0; i < 2; i++) { 921 if (((busid = apic_find_bus_id(bustype)) != -1) && 922 ((intrp = apic_find_io_intr_w_busid(irqno, busid)) 923 != NULL)) { 924 if ((newirq = apic_setup_irq_table(dip, irqno, 925 intrp, ispec, NULL, type)) != -1) { 926 return (newirq); 927 } 928 goto defconf; 929 } 930 bustype = (bustype == BUS_EISA) ? BUS_ISA : BUS_EISA; 931 } 932 } 933 934 /* MPS default configuration */ 935 defconf: 936 newirq = apic_setup_irq_table(dip, irqno, NULL, ispec, NULL, type); 937 if (newirq == -1) 938 return (-1); 939 ASSERT(IRQINDEX(newirq) == irqno); 940 ASSERT(apic_irq_table[irqno]); 941 return (newirq); 942 } 943 944 /* 945 * Attempt to share vector with someone else 946 */ 947 static int 948 apic_share_vector(int irqno, iflag_t *intr_flagp, short intr_index, int ipl, 949 uchar_t ioapicindex, uchar_t ipin, apic_irq_t **irqptrp) 950 { 951 #ifdef DEBUG 952 apic_irq_t *tmpirqp = NULL; 953 #endif /* DEBUG */ 954 apic_irq_t *irqptr, dummyirq; 955 int newirq, chosen_irq = -1, share = 127; 956 int lowest, highest, i; 957 uchar_t share_id; 958 959 DDI_INTR_IMPLDBG((CE_CONT, "apic_share_vector: irqno=0x%x " 960 "intr_index=0x%x ipl=0x%x\n", irqno, intr_index, ipl)); 961 962 highest = apic_ipltopri[ipl] + APIC_VECTOR_MASK; 963 lowest = apic_ipltopri[ipl-1] + APIC_VECTOR_PER_IPL; 964 965 if (highest < lowest) /* Both ipl and ipl-1 map to same pri */ 966 lowest -= APIC_VECTOR_PER_IPL; 967 dummyirq.airq_mps_intr_index = intr_index; 968 dummyirq.airq_ioapicindex = ioapicindex; 969 dummyirq.airq_intin_no = ipin; 970 if (intr_flagp) 971 dummyirq.airq_iflag = *intr_flagp; 972 apic_record_rdt_entry(&dummyirq, irqno); 973 for (i = lowest; i <= highest; i++) { 974 newirq = apic_vector_to_irq[i]; 975 if (newirq == APIC_RESV_IRQ) 976 continue; 977 irqptr = apic_irq_table[newirq]; 978 979 if ((dummyirq.airq_rdt_entry & 0xFF00) != 980 (irqptr->airq_rdt_entry & 0xFF00)) 981 /* not compatible */ 982 continue; 983 984 if (irqptr->airq_share < share) { 985 share = irqptr->airq_share; 986 chosen_irq = newirq; 987 } 988 } 989 if (chosen_irq != -1) { 990 /* 991 * Assign a share id which is free or which is larger 992 * than the largest one. 993 */ 994 share_id = 1; 995 mutex_enter(&airq_mutex); 996 irqptr = apic_irq_table[chosen_irq]; 997 while (irqptr) { 998 if (irqptr->airq_mps_intr_index == FREE_INDEX) { 999 share_id = irqptr->airq_share_id; 1000 break; 1001 } 1002 if (share_id <= irqptr->airq_share_id) 1003 share_id = irqptr->airq_share_id + 1; 1004 #ifdef DEBUG 1005 tmpirqp = irqptr; 1006 #endif /* DEBUG */ 1007 irqptr = irqptr->airq_next; 1008 } 1009 if (!irqptr) { 1010 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP); 1011 irqptr->airq_temp_cpu = IRQ_UNINIT; 1012 irqptr->airq_next = 1013 apic_irq_table[chosen_irq]->airq_next; 1014 apic_irq_table[chosen_irq]->airq_next = irqptr; 1015 #ifdef DEBUG 1016 tmpirqp = apic_irq_table[chosen_irq]; 1017 #endif /* DEBUG */ 1018 } 1019 irqptr->airq_mps_intr_index = intr_index; 1020 irqptr->airq_ioapicindex = ioapicindex; 1021 irqptr->airq_intin_no = ipin; 1022 if (intr_flagp) 1023 irqptr->airq_iflag = *intr_flagp; 1024 irqptr->airq_vector = apic_irq_table[chosen_irq]->airq_vector; 1025 irqptr->airq_share_id = share_id; 1026 apic_record_rdt_entry(irqptr, irqno); 1027 *irqptrp = irqptr; 1028 #ifdef DEBUG 1029 /* shuffle the pointers to test apic_delspl path */ 1030 if (tmpirqp) { 1031 tmpirqp->airq_next = irqptr->airq_next; 1032 irqptr->airq_next = apic_irq_table[chosen_irq]; 1033 apic_irq_table[chosen_irq] = irqptr; 1034 } 1035 #endif /* DEBUG */ 1036 mutex_exit(&airq_mutex); 1037 return (VIRTIRQ(chosen_irq, share_id)); 1038 } 1039 return (-1); 1040 } 1041 1042 /* 1043 * Allocate/Initialize the apic_irq_table[] entry for given irqno. If the entry 1044 * is used already, we will try to allocate a new irqno. 1045 * 1046 * Return value: 1047 * Success: irqno 1048 * Failure: -1 1049 */ 1050 static int 1051 apic_setup_irq_table(dev_info_t *dip, int irqno, struct apic_io_intr *intrp, 1052 struct intrspec *ispec, iflag_t *intr_flagp, int type) 1053 { 1054 int origirq; 1055 uchar_t ipl; 1056 int newirq, intr_index; 1057 uchar_t ipin, ioapic, ioapicindex, vector; 1058 apic_irq_t *irqptr; 1059 major_t major; 1060 dev_info_t *sdip; 1061 1062 ASSERT(ispec != NULL); 1063 1064 origirq = ispec->intrspec_vec; 1065 ipl = ispec->intrspec_pri; 1066 1067 DDI_INTR_IMPLDBG((CE_CONT, "apic_setup_irq_table: dip=0x%p type=%d " 1068 "irqno=0x%x origirq=0x%x\n", (void *)dip, type, irqno, origirq)); 1069 1070 major = (dip != NULL) ? ddi_driver_major(dip) : 0; 1071 1072 if (DDI_INTR_IS_MSI_OR_MSIX(type)) { 1073 /* MSI/X doesn't need to setup ioapic stuffs */ 1074 ioapicindex = 0xff; 1075 ioapic = 0xff; 1076 ipin = (uchar_t)0xff; 1077 intr_index = (type == DDI_INTR_TYPE_MSI) ? MSI_INDEX : 1078 MSIX_INDEX; 1079 mutex_enter(&airq_mutex); 1080 if ((irqno = apic_allocate_irq(apic_first_avail_irq)) == -1) { 1081 mutex_exit(&airq_mutex); 1082 /* need an irq for MSI/X to index into autovect[] */ 1083 cmn_err(CE_WARN, "No interrupt irq: %s instance %d", 1084 ddi_get_name(dip), ddi_get_instance(dip)); 1085 return (-1); 1086 } 1087 mutex_exit(&airq_mutex); 1088 1089 } else if (intrp != NULL) { 1090 intr_index = (int)(intrp - apic_io_intrp); 1091 ioapic = intrp->intr_destid; 1092 ipin = intrp->intr_destintin; 1093 /* Find ioapicindex. If destid was ALL, we will exit with 0. */ 1094 for (ioapicindex = apic_io_max - 1; ioapicindex; ioapicindex--) 1095 if (apic_io_id[ioapicindex] == ioapic) 1096 break; 1097 ASSERT((ioapic == apic_io_id[ioapicindex]) || 1098 (ioapic == INTR_ALL_APIC)); 1099 1100 /* check whether this intin# has been used by another irqno */ 1101 if ((newirq = apic_find_intin(ioapicindex, ipin)) != -1) { 1102 return (newirq); 1103 } 1104 1105 } else if (intr_flagp != NULL) { 1106 /* ACPI case */ 1107 intr_index = ACPI_INDEX; 1108 ioapicindex = acpi_find_ioapic(irqno); 1109 ASSERT(ioapicindex != 0xFF); 1110 ioapic = apic_io_id[ioapicindex]; 1111 ipin = irqno - apic_io_vectbase[ioapicindex]; 1112 if (apic_irq_table[irqno] && 1113 apic_irq_table[irqno]->airq_mps_intr_index == ACPI_INDEX) { 1114 ASSERT(apic_irq_table[irqno]->airq_intin_no == ipin && 1115 apic_irq_table[irqno]->airq_ioapicindex == 1116 ioapicindex); 1117 return (irqno); 1118 } 1119 1120 } else { 1121 /* default configuration */ 1122 ioapicindex = 0; 1123 ioapic = apic_io_id[ioapicindex]; 1124 ipin = (uchar_t)irqno; 1125 intr_index = DEFAULT_INDEX; 1126 } 1127 1128 if ((vector = apic_allocate_vector(ipl, irqno, 0)) == 0) { 1129 if ((newirq = apic_share_vector(irqno, intr_flagp, intr_index, 1130 ipl, ioapicindex, ipin, &irqptr)) != -1) { 1131 irqptr->airq_ipl = ipl; 1132 irqptr->airq_origirq = (uchar_t)origirq; 1133 irqptr->airq_dip = dip; 1134 irqptr->airq_major = major; 1135 sdip = apic_irq_table[IRQINDEX(newirq)]->airq_dip; 1136 /* This is OK to do really */ 1137 if (sdip == NULL) { 1138 cmn_err(CE_WARN, "Sharing vectors: %s" 1139 " instance %d and SCI", 1140 ddi_get_name(dip), ddi_get_instance(dip)); 1141 } else { 1142 cmn_err(CE_WARN, "Sharing vectors: %s" 1143 " instance %d and %s instance %d", 1144 ddi_get_name(sdip), ddi_get_instance(sdip), 1145 ddi_get_name(dip), ddi_get_instance(dip)); 1146 } 1147 return (newirq); 1148 } 1149 /* try high priority allocation now that share has failed */ 1150 if ((vector = apic_allocate_vector(ipl, irqno, 1)) == 0) { 1151 cmn_err(CE_WARN, "No interrupt vector: %s instance %d", 1152 ddi_get_name(dip), ddi_get_instance(dip)); 1153 return (-1); 1154 } 1155 } 1156 1157 mutex_enter(&airq_mutex); 1158 if (apic_irq_table[irqno] == NULL) { 1159 irqptr = kmem_zalloc(sizeof (apic_irq_t), KM_SLEEP); 1160 irqptr->airq_temp_cpu = IRQ_UNINIT; 1161 apic_irq_table[irqno] = irqptr; 1162 } else { 1163 irqptr = apic_irq_table[irqno]; 1164 if (irqptr->airq_mps_intr_index != FREE_INDEX) { 1165 /* 1166 * The slot is used by another irqno, so allocate 1167 * a free irqno for this interrupt 1168 */ 1169 newirq = apic_allocate_irq(apic_first_avail_irq); 1170 if (newirq == -1) { 1171 mutex_exit(&airq_mutex); 1172 return (-1); 1173 } 1174 irqno = newirq; 1175 irqptr = apic_irq_table[irqno]; 1176 if (irqptr == NULL) { 1177 irqptr = kmem_zalloc(sizeof (apic_irq_t), 1178 KM_SLEEP); 1179 irqptr->airq_temp_cpu = IRQ_UNINIT; 1180 apic_irq_table[irqno] = irqptr; 1181 } 1182 vector = apic_modify_vector(vector, newirq); 1183 } 1184 } 1185 apic_max_device_irq = max(irqno, apic_max_device_irq); 1186 apic_min_device_irq = min(irqno, apic_min_device_irq); 1187 mutex_exit(&airq_mutex); 1188 irqptr->airq_ioapicindex = ioapicindex; 1189 irqptr->airq_intin_no = ipin; 1190 irqptr->airq_ipl = ipl; 1191 irqptr->airq_vector = vector; 1192 irqptr->airq_origirq = (uchar_t)origirq; 1193 irqptr->airq_share_id = 0; 1194 irqptr->airq_mps_intr_index = (short)intr_index; 1195 irqptr->airq_dip = dip; 1196 irqptr->airq_major = major; 1197 irqptr->airq_cpu = apic_bind_intr(dip, irqno, ioapic, ipin); 1198 if (intr_flagp) 1199 irqptr->airq_iflag = *intr_flagp; 1200 1201 if (!DDI_INTR_IS_MSI_OR_MSIX(type)) { 1202 /* setup I/O APIC entry for non-MSI/X interrupts */ 1203 apic_record_rdt_entry(irqptr, irqno); 1204 } 1205 return (irqno); 1206 } 1207 1208 /* 1209 * return the cpu to which this intr should be bound. 1210 * Check properties or any other mechanism to see if user wants it 1211 * bound to a specific CPU. If so, return the cpu id with high bit set. 1212 * If not, use the policy to choose a cpu and return the id. 1213 */ 1214 uint32_t 1215 apic_bind_intr(dev_info_t *dip, int irq, uchar_t ioapicid, uchar_t intin) 1216 { 1217 int instance, instno, prop_len, bind_cpu, count; 1218 uint_t i, rc; 1219 uint32_t cpu; 1220 major_t major; 1221 char *name, *drv_name, *prop_val, *cptr; 1222 char prop_name[32]; 1223 ulong_t iflag; 1224 1225 1226 if (apic_intr_policy == INTR_LOWEST_PRIORITY) 1227 return (IRQ_UNBOUND); 1228 1229 if (apic_nproc == 1) 1230 return (0); 1231 1232 /* 1233 * dip may be NULL for interrupts not associated with a device driver, 1234 * such as the ACPI SCI or HPET interrupts. In that case just use the 1235 * next CPU and return. 1236 */ 1237 if (dip == NULL) { 1238 iflag = intr_clear(); 1239 lock_set(&apic_ioapic_lock); 1240 bind_cpu = apic_get_next_bind_cpu(); 1241 lock_clear(&apic_ioapic_lock); 1242 intr_restore(iflag); 1243 1244 cmn_err(CE_CONT, "!%s: irq 0x%x " 1245 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n", 1246 psm_name, irq, apic_irq_table[irq]->airq_vector, ioapicid, 1247 intin, bind_cpu & ~IRQ_USER_BOUND); 1248 1249 return ((uint32_t)bind_cpu); 1250 } 1251 1252 name = ddi_get_name(dip); 1253 major = ddi_name_to_major(name); 1254 drv_name = ddi_major_to_name(major); 1255 instance = ddi_get_instance(dip); 1256 if (apic_intr_policy == INTR_ROUND_ROBIN_WITH_AFFINITY) { 1257 i = apic_min_device_irq; 1258 for (; i <= apic_max_device_irq; i++) { 1259 if ((i == irq) || (apic_irq_table[i] == NULL) || 1260 (apic_irq_table[i]->airq_mps_intr_index 1261 == FREE_INDEX)) 1262 continue; 1263 1264 if ((apic_irq_table[i]->airq_major == major) && 1265 (!(apic_irq_table[i]->airq_cpu & IRQ_USER_BOUND))) { 1266 cpu = apic_irq_table[i]->airq_cpu; 1267 1268 cmn_err(CE_CONT, 1269 "!%s: %s (%s) instance #%d " 1270 "irq 0x%x vector 0x%x ioapic 0x%x " 1271 "intin 0x%x is bound to cpu %d\n", 1272 psm_name, 1273 name, drv_name, instance, irq, 1274 apic_irq_table[irq]->airq_vector, 1275 ioapicid, intin, cpu); 1276 return (cpu); 1277 } 1278 } 1279 } 1280 /* 1281 * search for "drvname"_intpt_bind_cpus property first, the 1282 * syntax of the property should be "a[,b,c,...]" where 1283 * instance 0 binds to cpu a, instance 1 binds to cpu b, 1284 * instance 3 binds to cpu c... 1285 * ddi_getlongprop() will search /option first, then / 1286 * if "drvname"_intpt_bind_cpus doesn't exist, then find 1287 * intpt_bind_cpus property. The syntax is the same, and 1288 * it applies to all the devices if its "drvname" specific 1289 * property doesn't exist 1290 */ 1291 (void) strcpy(prop_name, drv_name); 1292 (void) strcat(prop_name, "_intpt_bind_cpus"); 1293 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, prop_name, 1294 (caddr_t)&prop_val, &prop_len); 1295 if (rc != DDI_PROP_SUCCESS) { 1296 rc = ddi_getlongprop(DDI_DEV_T_ANY, dip, 0, 1297 "intpt_bind_cpus", (caddr_t)&prop_val, &prop_len); 1298 } 1299 if (rc == DDI_PROP_SUCCESS) { 1300 for (i = count = 0; i < (prop_len - 1); i++) 1301 if (prop_val[i] == ',') 1302 count++; 1303 if (prop_val[i-1] != ',') 1304 count++; 1305 /* 1306 * if somehow the binding instances defined in the 1307 * property are not enough for this instno., then 1308 * reuse the pattern for the next instance until 1309 * it reaches the requested instno 1310 */ 1311 instno = instance % count; 1312 i = 0; 1313 cptr = prop_val; 1314 while (i < instno) 1315 if (*cptr++ == ',') 1316 i++; 1317 bind_cpu = stoi(&cptr); 1318 kmem_free(prop_val, prop_len); 1319 /* if specific CPU is bogus, then default to next cpu */ 1320 if (!apic_cpu_in_range(bind_cpu)) { 1321 cmn_err(CE_WARN, "%s: %s=%s: CPU %d not present", 1322 psm_name, prop_name, prop_val, bind_cpu); 1323 rc = DDI_PROP_NOT_FOUND; 1324 } else { 1325 /* indicate that we are bound at user request */ 1326 bind_cpu |= IRQ_USER_BOUND; 1327 } 1328 /* 1329 * no need to check apic_cpus[].aci_status, if specific CPU is 1330 * not up, then post_cpu_start will handle it. 1331 */ 1332 } 1333 1334 if (rc != DDI_PROP_SUCCESS) { 1335 iflag = intr_clear(); 1336 lock_set(&apic_ioapic_lock); 1337 bind_cpu = apic_get_next_bind_cpu(); 1338 lock_clear(&apic_ioapic_lock); 1339 intr_restore(iflag); 1340 } 1341 1342 cmn_err(CE_CONT, "!%s: %s (%s) instance %d irq 0x%x " 1343 "vector 0x%x ioapic 0x%x intin 0x%x is bound to cpu %d\n", 1344 psm_name, name, drv_name, instance, irq, 1345 apic_irq_table[irq]->airq_vector, ioapicid, intin, 1346 bind_cpu & ~IRQ_USER_BOUND); 1347 1348 return ((uint32_t)bind_cpu); 1349 } 1350 1351 /* 1352 * Mark vector as being in the process of being deleted. Interrupts 1353 * may still come in on some CPU. The moment an interrupt comes with 1354 * the new vector, we know we can free the old one. Called only from 1355 * addspl and delspl with interrupts disabled. Because an interrupt 1356 * can be shared, but no interrupt from either device may come in, 1357 * we also use a timeout mechanism, which we arbitrarily set to 1358 * apic_revector_timeout microseconds. 1359 */ 1360 static void 1361 apic_mark_vector(uchar_t oldvector, uchar_t newvector) 1362 { 1363 ulong_t iflag; 1364 1365 iflag = intr_clear(); 1366 lock_set(&apic_revector_lock); 1367 if (!apic_oldvec_to_newvec) { 1368 apic_oldvec_to_newvec = 1369 kmem_zalloc(sizeof (newvector) * APIC_MAX_VECTOR * 2, 1370 KM_NOSLEEP); 1371 1372 if (!apic_oldvec_to_newvec) { 1373 /* 1374 * This failure is not catastrophic. 1375 * But, the oldvec will never be freed. 1376 */ 1377 apic_error |= APIC_ERR_MARK_VECTOR_FAIL; 1378 lock_clear(&apic_revector_lock); 1379 intr_restore(iflag); 1380 return; 1381 } 1382 apic_newvec_to_oldvec = &apic_oldvec_to_newvec[APIC_MAX_VECTOR]; 1383 } 1384 1385 /* See if we already did this for drivers which do double addintrs */ 1386 if (apic_oldvec_to_newvec[oldvector] != newvector) { 1387 apic_oldvec_to_newvec[oldvector] = newvector; 1388 apic_newvec_to_oldvec[newvector] = oldvector; 1389 apic_revector_pending++; 1390 } 1391 lock_clear(&apic_revector_lock); 1392 intr_restore(iflag); 1393 (void) timeout(apic_xlate_vector_free_timeout_handler, 1394 (void *)(uintptr_t)oldvector, drv_usectohz(apic_revector_timeout)); 1395 } 1396 1397 /* 1398 * xlate_vector is called from intr_enter if revector_pending is set. 1399 * It will xlate it if needed and mark the old vector as free. 1400 */ 1401 uchar_t 1402 apic_xlate_vector(uchar_t vector) 1403 { 1404 uchar_t newvector, oldvector = 0; 1405 1406 lock_set(&apic_revector_lock); 1407 /* Do we really need to do this ? */ 1408 if (!apic_revector_pending) { 1409 lock_clear(&apic_revector_lock); 1410 return (vector); 1411 } 1412 if ((newvector = apic_oldvec_to_newvec[vector]) != 0) 1413 oldvector = vector; 1414 else { 1415 /* 1416 * The incoming vector is new . See if a stale entry is 1417 * remaining 1418 */ 1419 if ((oldvector = apic_newvec_to_oldvec[vector]) != 0) 1420 newvector = vector; 1421 } 1422 1423 if (oldvector) { 1424 apic_revector_pending--; 1425 apic_oldvec_to_newvec[oldvector] = 0; 1426 apic_newvec_to_oldvec[newvector] = 0; 1427 apic_free_vector(oldvector); 1428 lock_clear(&apic_revector_lock); 1429 /* There could have been more than one reprogramming! */ 1430 return (apic_xlate_vector(newvector)); 1431 } 1432 lock_clear(&apic_revector_lock); 1433 return (vector); 1434 } 1435 1436 void 1437 apic_xlate_vector_free_timeout_handler(void *arg) 1438 { 1439 ulong_t iflag; 1440 uchar_t oldvector, newvector; 1441 1442 oldvector = (uchar_t)(uintptr_t)arg; 1443 iflag = intr_clear(); 1444 lock_set(&apic_revector_lock); 1445 if ((newvector = apic_oldvec_to_newvec[oldvector]) != 0) { 1446 apic_free_vector(oldvector); 1447 apic_oldvec_to_newvec[oldvector] = 0; 1448 apic_newvec_to_oldvec[newvector] = 0; 1449 apic_revector_pending--; 1450 } 1451 1452 lock_clear(&apic_revector_lock); 1453 intr_restore(iflag); 1454 } 1455 1456 /* 1457 * Bind interrupt corresponding to irq_ptr to bind_cpu. 1458 * Must be called with interrupts disabled and apic_ioapic_lock held 1459 */ 1460 int 1461 apic_rebind(apic_irq_t *irq_ptr, int bind_cpu, 1462 struct ioapic_reprogram_data *drep) 1463 { 1464 int ioapicindex, intin_no; 1465 uint32_t airq_temp_cpu; 1466 apic_cpus_info_t *cpu_infop; 1467 uint32_t rdt_entry; 1468 int which_irq; 1469 ioapic_rdt_t irdt; 1470 1471 which_irq = apic_vector_to_irq[irq_ptr->airq_vector]; 1472 1473 intin_no = irq_ptr->airq_intin_no; 1474 ioapicindex = irq_ptr->airq_ioapicindex; 1475 airq_temp_cpu = irq_ptr->airq_temp_cpu; 1476 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != IRQ_UNBOUND) { 1477 if (airq_temp_cpu & IRQ_USER_BOUND) 1478 /* Mask off high bit so it can be used as array index */ 1479 airq_temp_cpu &= ~IRQ_USER_BOUND; 1480 1481 ASSERT(apic_cpu_in_range(airq_temp_cpu)); 1482 } 1483 1484 /* 1485 * Can't bind to a CPU that's not accepting interrupts: 1486 */ 1487 cpu_infop = &apic_cpus[bind_cpu & ~IRQ_USER_BOUND]; 1488 if (!(cpu_infop->aci_status & APIC_CPU_INTR_ENABLE)) 1489 return (1); 1490 1491 /* 1492 * If we are about to change the interrupt vector for this interrupt, 1493 * and this interrupt is level-triggered, attached to an IOAPIC, 1494 * has been delivered to a CPU and that CPU has not handled it 1495 * yet, we cannot reprogram the IOAPIC now. 1496 */ 1497 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) { 1498 1499 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, 1500 intin_no); 1501 1502 if ((irq_ptr->airq_vector != RDT_VECTOR(rdt_entry)) && 1503 apic_check_stuck_interrupt(irq_ptr, airq_temp_cpu, 1504 bind_cpu, ioapicindex, intin_no, which_irq, drep) != 0) { 1505 1506 return (0); 1507 } 1508 1509 /* 1510 * NOTE: We do not unmask the RDT here, as an interrupt MAY 1511 * still come in before we have a chance to reprogram it below. 1512 * The reprogramming below will simultaneously change and 1513 * unmask the RDT entry. 1514 */ 1515 1516 if ((uint32_t)bind_cpu == IRQ_UNBOUND) { 1517 irdt.ir_lo = AV_LDEST | AV_LOPRI | 1518 irq_ptr->airq_rdt_entry; 1519 1520 irdt.ir_hi = AV_TOALL >> APIC_ID_BIT_OFFSET; 1521 1522 apic_vt_ops->apic_intrmap_alloc_entry( 1523 &irq_ptr->airq_intrmap_private, NULL, 1524 DDI_INTR_TYPE_FIXED, 1, ioapicindex); 1525 apic_vt_ops->apic_intrmap_map_entry( 1526 irq_ptr->airq_intrmap_private, (void *)&irdt, 1527 DDI_INTR_TYPE_FIXED, 1); 1528 apic_vt_ops->apic_intrmap_record_rdt( 1529 irq_ptr->airq_intrmap_private, &irdt); 1530 1531 /* Write the RDT entry -- no specific CPU binding */ 1532 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no, 1533 irdt.ir_hi | AV_TOALL); 1534 1535 if (airq_temp_cpu != IRQ_UNINIT && airq_temp_cpu != 1536 IRQ_UNBOUND) 1537 apic_cpus[airq_temp_cpu].aci_temp_bound--; 1538 1539 /* 1540 * Write the vector, trigger, and polarity portion of 1541 * the RDT 1542 */ 1543 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no, 1544 irdt.ir_lo); 1545 1546 irq_ptr->airq_temp_cpu = IRQ_UNBOUND; 1547 return (0); 1548 } 1549 } 1550 1551 if (bind_cpu & IRQ_USER_BOUND) { 1552 cpu_infop->aci_bound++; 1553 } else { 1554 cpu_infop->aci_temp_bound++; 1555 } 1556 ASSERT(apic_cpu_in_range(bind_cpu)); 1557 1558 if ((airq_temp_cpu != IRQ_UNBOUND) && (airq_temp_cpu != IRQ_UNINIT)) { 1559 apic_cpus[airq_temp_cpu].aci_temp_bound--; 1560 } 1561 if (!APIC_IS_MSI_OR_MSIX_INDEX(irq_ptr->airq_mps_intr_index)) { 1562 1563 irdt.ir_lo = AV_PDEST | AV_FIXED | irq_ptr->airq_rdt_entry; 1564 irdt.ir_hi = cpu_infop->aci_local_id; 1565 1566 apic_vt_ops->apic_intrmap_alloc_entry( 1567 &irq_ptr->airq_intrmap_private, NULL, DDI_INTR_TYPE_FIXED, 1568 1, ioapicindex); 1569 apic_vt_ops->apic_intrmap_map_entry( 1570 irq_ptr->airq_intrmap_private, 1571 (void *)&irdt, DDI_INTR_TYPE_FIXED, 1); 1572 apic_vt_ops->apic_intrmap_record_rdt( 1573 irq_ptr->airq_intrmap_private, &irdt); 1574 1575 /* Write the RDT entry -- bind to a specific CPU: */ 1576 WRITE_IOAPIC_RDT_ENTRY_HIGH_DWORD(ioapicindex, intin_no, 1577 irdt.ir_hi); 1578 1579 /* Write the vector, trigger, and polarity portion of the RDT */ 1580 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapicindex, intin_no, 1581 irdt.ir_lo); 1582 1583 } else { 1584 int type = (irq_ptr->airq_mps_intr_index == MSI_INDEX) ? 1585 DDI_INTR_TYPE_MSI : DDI_INTR_TYPE_MSIX; 1586 if (type == DDI_INTR_TYPE_MSI) { 1587 if (irq_ptr->airq_ioapicindex == 1588 irq_ptr->airq_origirq) { 1589 /* first one */ 1590 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call " 1591 "apic_pci_msi_enable_vector\n")); 1592 apic_pci_msi_enable_vector(irq_ptr, 1593 type, which_irq, irq_ptr->airq_vector, 1594 irq_ptr->airq_intin_no, 1595 cpu_infop->aci_local_id); 1596 } 1597 if ((irq_ptr->airq_ioapicindex + 1598 irq_ptr->airq_intin_no - 1) == 1599 irq_ptr->airq_origirq) { /* last one */ 1600 DDI_INTR_IMPLDBG((CE_CONT, "apic_rebind: call " 1601 "apic_pci_msi_enable_mode\n")); 1602 apic_pci_msi_enable_mode(irq_ptr->airq_dip, 1603 type, which_irq); 1604 } 1605 } else { /* MSI-X */ 1606 apic_pci_msi_enable_vector(irq_ptr, type, 1607 irq_ptr->airq_origirq, irq_ptr->airq_vector, 1, 1608 cpu_infop->aci_local_id); 1609 apic_pci_msi_enable_mode(irq_ptr->airq_dip, type, 1610 irq_ptr->airq_origirq); 1611 } 1612 } 1613 irq_ptr->airq_temp_cpu = (uint32_t)bind_cpu; 1614 apic_redist_cpu_skip &= ~(1 << (bind_cpu & ~IRQ_USER_BOUND)); 1615 return (0); 1616 } 1617 1618 static void 1619 apic_last_ditch_clear_remote_irr(int ioapic_ix, int intin_no) 1620 { 1621 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) 1622 & AV_REMOTE_IRR) != 0) { 1623 /* 1624 * Trying to clear the bit through normal 1625 * channels has failed. So as a last-ditch 1626 * effort, try to set the trigger mode to 1627 * edge, then to level. This has been 1628 * observed to work on many systems. 1629 */ 1630 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1631 intin_no, 1632 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1633 intin_no) & ~AV_LEVEL); 1634 1635 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1636 intin_no, 1637 READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1638 intin_no) | AV_LEVEL); 1639 1640 /* 1641 * If the bit's STILL set, this interrupt may 1642 * be hosed. 1643 */ 1644 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1645 intin_no) & AV_REMOTE_IRR) != 0) { 1646 1647 prom_printf("%s: Remote IRR still " 1648 "not clear for IOAPIC %d intin %d.\n" 1649 "\tInterrupts to this pin may cease " 1650 "functioning.\n", psm_name, ioapic_ix, 1651 intin_no); 1652 #ifdef DEBUG 1653 apic_last_ditch_reprogram_failures++; 1654 #endif 1655 } 1656 } 1657 } 1658 1659 /* 1660 * This function is protected by apic_ioapic_lock coupled with the 1661 * fact that interrupts are disabled. 1662 */ 1663 static void 1664 delete_defer_repro_ent(int which_irq) 1665 { 1666 ASSERT(which_irq >= 0); 1667 ASSERT(which_irq <= 255); 1668 ASSERT(LOCK_HELD(&apic_ioapic_lock)); 1669 1670 if (apic_reprogram_info[which_irq].done) 1671 return; 1672 1673 apic_reprogram_info[which_irq].done = B_TRUE; 1674 1675 #ifdef DEBUG 1676 apic_defer_repro_total_retries += 1677 apic_reprogram_info[which_irq].tries; 1678 1679 apic_defer_repro_successes++; 1680 #endif 1681 1682 if (--apic_reprogram_outstanding == 0) { 1683 1684 setlvlx = psm_intr_exit_fn(); 1685 } 1686 } 1687 1688 1689 /* 1690 * Interrupts must be disabled during this function to prevent 1691 * self-deadlock. Interrupts are disabled because this function 1692 * is called from apic_check_stuck_interrupt(), which is called 1693 * from apic_rebind(), which requires its caller to disable interrupts. 1694 */ 1695 static void 1696 add_defer_repro_ent(apic_irq_t *irq_ptr, int which_irq, int new_bind_cpu) 1697 { 1698 ASSERT(which_irq >= 0); 1699 ASSERT(which_irq <= 255); 1700 ASSERT(!interrupts_enabled()); 1701 1702 /* 1703 * On the off-chance that there's already a deferred 1704 * reprogramming on this irq, check, and if so, just update the 1705 * CPU and irq pointer to which the interrupt is targeted, then return. 1706 */ 1707 if (!apic_reprogram_info[which_irq].done) { 1708 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu; 1709 apic_reprogram_info[which_irq].irqp = irq_ptr; 1710 return; 1711 } 1712 1713 apic_reprogram_info[which_irq].irqp = irq_ptr; 1714 apic_reprogram_info[which_irq].bindcpu = new_bind_cpu; 1715 apic_reprogram_info[which_irq].tries = 0; 1716 /* 1717 * This must be the last thing set, since we're not 1718 * grabbing any locks, apic_try_deferred_reprogram() will 1719 * make its decision about using this entry iff done 1720 * is false. 1721 */ 1722 apic_reprogram_info[which_irq].done = B_FALSE; 1723 1724 /* 1725 * If there were previously no deferred reprogrammings, change 1726 * setlvlx to call apic_try_deferred_reprogram() 1727 */ 1728 if (++apic_reprogram_outstanding == 1) { 1729 1730 setlvlx = apic_try_deferred_reprogram; 1731 } 1732 } 1733 1734 static void 1735 apic_try_deferred_reprogram(int prev_ipl, int irq) 1736 { 1737 int reproirq; 1738 ulong_t iflag; 1739 struct ioapic_reprogram_data *drep; 1740 1741 (*psm_intr_exit_fn())(prev_ipl, irq); 1742 1743 if (!lock_try(&apic_defer_reprogram_lock)) { 1744 return; 1745 } 1746 1747 /* 1748 * Acquire the apic_ioapic_lock so that any other operations that 1749 * may affect the apic_reprogram_info state are serialized. 1750 * It's still possible for the last deferred reprogramming to clear 1751 * between the time we entered this function and the time we get to 1752 * the for loop below. In that case, *setlvlx will have been set 1753 * back to *_intr_exit and drep will be NULL. (There's no way to 1754 * stop that from happening -- we would need to grab a lock before 1755 * calling *setlvlx, which is neither realistic nor prudent). 1756 */ 1757 iflag = intr_clear(); 1758 lock_set(&apic_ioapic_lock); 1759 1760 /* 1761 * For each deferred RDT entry, try to reprogram it now. Note that 1762 * there is no lock acquisition to read apic_reprogram_info because 1763 * '.done' is set only after the other fields in the structure are set. 1764 */ 1765 1766 drep = NULL; 1767 for (reproirq = 0; reproirq <= APIC_MAX_VECTOR; reproirq++) { 1768 if (apic_reprogram_info[reproirq].done == B_FALSE) { 1769 drep = &apic_reprogram_info[reproirq]; 1770 break; 1771 } 1772 } 1773 1774 /* 1775 * Either we found a deferred action to perform, or 1776 * we entered this function spuriously, after *setlvlx 1777 * was restored to point to *_intr_exit. Any other 1778 * permutation is invalid. 1779 */ 1780 ASSERT(drep != NULL || *setlvlx == psm_intr_exit_fn()); 1781 1782 /* 1783 * Though we can't really do anything about errors 1784 * at this point, keep track of them for reporting. 1785 * Note that it is very possible for apic_setup_io_intr 1786 * to re-register this very timeout if the Remote IRR bit 1787 * has not yet cleared. 1788 */ 1789 1790 #ifdef DEBUG 1791 if (drep != NULL) { 1792 if (apic_setup_io_intr(drep, reproirq, B_TRUE) != 0) { 1793 apic_deferred_setup_failures++; 1794 } 1795 } else { 1796 apic_deferred_spurious_enters++; 1797 } 1798 #else 1799 if (drep != NULL) 1800 (void) apic_setup_io_intr(drep, reproirq, B_TRUE); 1801 #endif 1802 1803 lock_clear(&apic_ioapic_lock); 1804 intr_restore(iflag); 1805 1806 lock_clear(&apic_defer_reprogram_lock); 1807 } 1808 1809 static void 1810 apic_ioapic_wait_pending_clear(int ioapic_ix, int intin_no) 1811 { 1812 int waited; 1813 1814 /* 1815 * Wait for the delivery pending bit to clear. 1816 */ 1817 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) & 1818 (AV_LEVEL|AV_PENDING)) == (AV_LEVEL|AV_PENDING)) { 1819 1820 /* 1821 * If we're still waiting on the delivery of this interrupt, 1822 * continue to wait here until it is delivered (this should be 1823 * a very small amount of time, but include a timeout just in 1824 * case). 1825 */ 1826 for (waited = 0; waited < apic_max_reps_clear_pending; 1827 waited++) { 1828 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1829 intin_no) & AV_PENDING) == 0) { 1830 break; 1831 } 1832 } 1833 } 1834 } 1835 1836 1837 /* 1838 * Checks to see if the IOAPIC interrupt entry specified has its Remote IRR 1839 * bit set. Calls functions that modify the function that setlvlx points to, 1840 * so that the reprogramming can be retried very shortly. 1841 * 1842 * This function will mask the RDT entry if the interrupt is level-triggered. 1843 * (The caller is responsible for unmasking the RDT entry.) 1844 * 1845 * Returns non-zero if the caller should defer IOAPIC reprogramming. 1846 */ 1847 static int 1848 apic_check_stuck_interrupt(apic_irq_t *irq_ptr, int old_bind_cpu, 1849 int new_bind_cpu, int ioapic_ix, int intin_no, int which_irq, 1850 struct ioapic_reprogram_data *drep) 1851 { 1852 int32_t rdt_entry; 1853 int waited; 1854 int reps = 0; 1855 1856 /* 1857 * Wait for the delivery pending bit to clear. 1858 */ 1859 do { 1860 ++reps; 1861 1862 apic_ioapic_wait_pending_clear(ioapic_ix, intin_no); 1863 1864 /* 1865 * Mask the RDT entry, but only if it's a level-triggered 1866 * interrupt 1867 */ 1868 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1869 intin_no); 1870 if ((rdt_entry & (AV_LEVEL|AV_MASK)) == AV_LEVEL) { 1871 1872 /* Mask it */ 1873 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no, 1874 AV_MASK | rdt_entry); 1875 } 1876 1877 if ((rdt_entry & AV_LEVEL) == AV_LEVEL) { 1878 /* 1879 * If there was a race and an interrupt was injected 1880 * just before we masked, check for that case here. 1881 * Then, unmask the RDT entry and try again. If we're 1882 * on our last try, don't unmask (because we want the 1883 * RDT entry to remain masked for the rest of the 1884 * function). 1885 */ 1886 rdt_entry = READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1887 intin_no); 1888 if ((rdt_entry & AV_PENDING) && 1889 (reps < apic_max_reps_clear_pending)) { 1890 /* Unmask it */ 1891 WRITE_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1892 intin_no, rdt_entry & ~AV_MASK); 1893 } 1894 } 1895 1896 } while ((rdt_entry & AV_PENDING) && 1897 (reps < apic_max_reps_clear_pending)); 1898 1899 #ifdef DEBUG 1900 if (rdt_entry & AV_PENDING) 1901 apic_intr_deliver_timeouts++; 1902 #endif 1903 1904 /* 1905 * If the remote IRR bit is set, then the interrupt has been sent 1906 * to a CPU for processing. We have no choice but to wait for 1907 * that CPU to process the interrupt, at which point the remote IRR 1908 * bit will be cleared. 1909 */ 1910 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, intin_no) & 1911 (AV_LEVEL|AV_REMOTE_IRR)) == (AV_LEVEL|AV_REMOTE_IRR)) { 1912 1913 /* 1914 * If the CPU that this RDT is bound to is NOT the current 1915 * CPU, wait until that CPU handles the interrupt and ACKs 1916 * it. If this interrupt is not bound to any CPU (that is, 1917 * if it's bound to the logical destination of "anyone"), it 1918 * may have been delivered to the current CPU so handle that 1919 * case by deferring the reprogramming (below). 1920 */ 1921 if ((old_bind_cpu != IRQ_UNBOUND) && 1922 (old_bind_cpu != IRQ_UNINIT) && 1923 (old_bind_cpu != psm_get_cpu_id())) { 1924 for (waited = 0; waited < apic_max_reps_clear_pending; 1925 waited++) { 1926 if ((READ_IOAPIC_RDT_ENTRY_LOW_DWORD(ioapic_ix, 1927 intin_no) & AV_REMOTE_IRR) == 0) { 1928 1929 delete_defer_repro_ent(which_irq); 1930 1931 /* Remote IRR has cleared! */ 1932 return (0); 1933 } 1934 } 1935 } 1936 1937 /* 1938 * If we waited and the Remote IRR bit is still not cleared, 1939 * AND if we've invoked the timeout APIC_REPROGRAM_MAX_TIMEOUTS 1940 * times for this interrupt, try the last-ditch workaround: 1941 */ 1942 if (drep && drep->tries >= APIC_REPROGRAM_MAX_TRIES) { 1943 1944 apic_last_ditch_clear_remote_irr(ioapic_ix, intin_no); 1945 1946 /* Mark this one as reprogrammed: */ 1947 delete_defer_repro_ent(which_irq); 1948 1949 return (0); 1950 } else { 1951 #ifdef DEBUG 1952 apic_intr_deferrals++; 1953 #endif 1954 1955 /* 1956 * If waiting for the Remote IRR bit (above) didn't 1957 * allow it to clear, defer the reprogramming. 1958 * Add a new deferred-programming entry if the 1959 * caller passed a NULL one (and update the existing one 1960 * in case anything changed). 1961 */ 1962 add_defer_repro_ent(irq_ptr, which_irq, new_bind_cpu); 1963 if (drep) 1964 drep->tries++; 1965 1966 /* Inform caller to defer IOAPIC programming: */ 1967 return (1); 1968 } 1969 1970 } 1971 1972 /* Remote IRR is clear */ 1973 delete_defer_repro_ent(which_irq); 1974 1975 return (0); 1976 } 1977 1978 /* 1979 * Called to migrate all interrupts at an irq to another cpu. 1980 * Must be called with interrupts disabled and apic_ioapic_lock held 1981 */ 1982 int 1983 apic_rebind_all(apic_irq_t *irq_ptr, int bind_cpu) 1984 { 1985 apic_irq_t *irqptr = irq_ptr; 1986 int retval = 0; 1987 1988 while (irqptr) { 1989 if (irqptr->airq_temp_cpu != IRQ_UNINIT) 1990 retval |= apic_rebind(irqptr, bind_cpu, NULL); 1991 irqptr = irqptr->airq_next; 1992 } 1993 1994 return (retval); 1995 } 1996 1997 /* 1998 * apic_intr_redistribute does all the messy computations for identifying 1999 * which interrupt to move to which CPU. Currently we do just one interrupt 2000 * at a time. This reduces the time we spent doing all this within clock 2001 * interrupt. When it is done in idle, we could do more than 1. 2002 * First we find the most busy and the most free CPU (time in ISR only) 2003 * skipping those CPUs that has been identified as being ineligible (cpu_skip) 2004 * Then we look for IRQs which are closest to the difference between the 2005 * most busy CPU and the average ISR load. We try to find one whose load 2006 * is less than difference.If none exists, then we chose one larger than the 2007 * difference, provided it does not make the most idle CPU worse than the 2008 * most busy one. In the end, we clear all the busy fields for CPUs. For 2009 * IRQs, they are cleared as they are scanned. 2010 */ 2011 void 2012 apic_intr_redistribute(void) 2013 { 2014 int busiest_cpu, most_free_cpu; 2015 int cpu_free, cpu_busy, max_busy, min_busy; 2016 int min_free, diff; 2017 int average_busy, cpus_online; 2018 int i, busy; 2019 ulong_t iflag; 2020 apic_cpus_info_t *cpu_infop; 2021 apic_irq_t *min_busy_irq = NULL; 2022 apic_irq_t *max_busy_irq = NULL; 2023 2024 busiest_cpu = most_free_cpu = -1; 2025 cpu_free = cpu_busy = max_busy = average_busy = 0; 2026 min_free = apic_sample_factor_redistribution; 2027 cpus_online = 0; 2028 /* 2029 * Below we will check for CPU_INTR_ENABLE, bound, temp_bound, temp_cpu 2030 * without ioapic_lock. That is OK as we are just doing statistical 2031 * sampling anyway and any inaccuracy now will get corrected next time 2032 * The call to rebind which actually changes things will make sure 2033 * we are consistent. 2034 */ 2035 for (i = 0; i < apic_nproc; i++) { 2036 if (apic_cpu_in_range(i) && 2037 !(apic_redist_cpu_skip & (1 << i)) && 2038 (apic_cpus[i].aci_status & APIC_CPU_INTR_ENABLE)) { 2039 2040 cpu_infop = &apic_cpus[i]; 2041 /* 2042 * If no unbound interrupts or only 1 total on this 2043 * CPU, skip 2044 */ 2045 if (!cpu_infop->aci_temp_bound || 2046 (cpu_infop->aci_bound + cpu_infop->aci_temp_bound) 2047 == 1) { 2048 apic_redist_cpu_skip |= 1 << i; 2049 continue; 2050 } 2051 2052 busy = cpu_infop->aci_busy; 2053 average_busy += busy; 2054 cpus_online++; 2055 if (max_busy < busy) { 2056 max_busy = busy; 2057 busiest_cpu = i; 2058 } 2059 if (min_free > busy) { 2060 min_free = busy; 2061 most_free_cpu = i; 2062 } 2063 if (busy > apic_int_busy_mark) { 2064 cpu_busy |= 1 << i; 2065 } else { 2066 if (busy < apic_int_free_mark) 2067 cpu_free |= 1 << i; 2068 } 2069 } 2070 } 2071 if ((cpu_busy && cpu_free) || 2072 (max_busy >= (min_free + apic_diff_for_redistribution))) { 2073 2074 apic_num_imbalance++; 2075 #ifdef DEBUG 2076 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) { 2077 prom_printf( 2078 "redistribute busy=%x free=%x max=%x min=%x", 2079 cpu_busy, cpu_free, max_busy, min_free); 2080 } 2081 #endif /* DEBUG */ 2082 2083 2084 average_busy /= cpus_online; 2085 2086 diff = max_busy - average_busy; 2087 min_busy = max_busy; /* start with the max possible value */ 2088 max_busy = 0; 2089 min_busy_irq = max_busy_irq = NULL; 2090 i = apic_min_device_irq; 2091 for (; i <= apic_max_device_irq; i++) { 2092 apic_irq_t *irq_ptr; 2093 /* Change to linked list per CPU ? */ 2094 if ((irq_ptr = apic_irq_table[i]) == NULL) 2095 continue; 2096 /* Check for irq_busy & decide which one to move */ 2097 /* Also zero them for next round */ 2098 if ((irq_ptr->airq_temp_cpu == busiest_cpu) && 2099 irq_ptr->airq_busy) { 2100 if (irq_ptr->airq_busy < diff) { 2101 /* 2102 * Check for least busy CPU, 2103 * best fit or what ? 2104 */ 2105 if (max_busy < irq_ptr->airq_busy) { 2106 /* 2107 * Most busy within the 2108 * required differential 2109 */ 2110 max_busy = irq_ptr->airq_busy; 2111 max_busy_irq = irq_ptr; 2112 } 2113 } else { 2114 if (min_busy > irq_ptr->airq_busy) { 2115 /* 2116 * least busy, but more than 2117 * the reqd diff 2118 */ 2119 if (min_busy < 2120 (diff + average_busy - 2121 min_free)) { 2122 /* 2123 * Making sure new cpu 2124 * will not end up 2125 * worse 2126 */ 2127 min_busy = 2128 irq_ptr->airq_busy; 2129 2130 min_busy_irq = irq_ptr; 2131 } 2132 } 2133 } 2134 } 2135 irq_ptr->airq_busy = 0; 2136 } 2137 2138 if (max_busy_irq != NULL) { 2139 #ifdef DEBUG 2140 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) { 2141 prom_printf("rebinding %x to %x", 2142 max_busy_irq->airq_vector, most_free_cpu); 2143 } 2144 #endif /* DEBUG */ 2145 iflag = intr_clear(); 2146 if (lock_try(&apic_ioapic_lock)) { 2147 if (apic_rebind_all(max_busy_irq, 2148 most_free_cpu) == 0) { 2149 /* Make change permenant */ 2150 max_busy_irq->airq_cpu = 2151 (uint32_t)most_free_cpu; 2152 } 2153 lock_clear(&apic_ioapic_lock); 2154 } 2155 intr_restore(iflag); 2156 2157 } else if (min_busy_irq != NULL) { 2158 #ifdef DEBUG 2159 if (apic_verbose & APIC_VERBOSE_IOAPIC_FLAG) { 2160 prom_printf("rebinding %x to %x", 2161 min_busy_irq->airq_vector, most_free_cpu); 2162 } 2163 #endif /* DEBUG */ 2164 2165 iflag = intr_clear(); 2166 if (lock_try(&apic_ioapic_lock)) { 2167 if (apic_rebind_all(min_busy_irq, 2168 most_free_cpu) == 0) { 2169 /* Make change permenant */ 2170 min_busy_irq->airq_cpu = 2171 (uint32_t)most_free_cpu; 2172 } 2173 lock_clear(&apic_ioapic_lock); 2174 } 2175 intr_restore(iflag); 2176 2177 } else { 2178 if (cpu_busy != (1 << busiest_cpu)) { 2179 apic_redist_cpu_skip |= 1 << busiest_cpu; 2180 /* 2181 * We leave cpu_skip set so that next time we 2182 * can choose another cpu 2183 */ 2184 } 2185 } 2186 apic_num_rebind++; 2187 } else { 2188 /* 2189 * found nothing. Could be that we skipped over valid CPUs 2190 * or we have balanced everything. If we had a variable 2191 * ticks_for_redistribution, it could be increased here. 2192 * apic_int_busy, int_free etc would also need to be 2193 * changed. 2194 */ 2195 if (apic_redist_cpu_skip) 2196 apic_redist_cpu_skip = 0; 2197 } 2198 for (i = 0; i < apic_nproc; i++) { 2199 if (apic_cpu_in_range(i)) { 2200 apic_cpus[i].aci_busy = 0; 2201 } 2202 } 2203 } 2204 2205 void 2206 apic_cleanup_busy(void) 2207 { 2208 int i; 2209 apic_irq_t *irq_ptr; 2210 2211 for (i = 0; i < apic_nproc; i++) { 2212 if (apic_cpu_in_range(i)) { 2213 apic_cpus[i].aci_busy = 0; 2214 } 2215 } 2216 2217 for (i = apic_min_device_irq; i <= apic_max_device_irq; i++) { 2218 if ((irq_ptr = apic_irq_table[i]) != NULL) 2219 irq_ptr->airq_busy = 0; 2220 } 2221 } 2222