1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/cpuvar.h> 27 #include <sys/psm.h> 28 #include <sys/archsystm.h> 29 #include <sys/apic.h> 30 #include <sys/sunddi.h> 31 #include <sys/ddi_impldefs.h> 32 #include <sys/mach_intr.h> 33 #include <sys/sysmacros.h> 34 #include <sys/trap.h> 35 #include <sys/x86_archext.h> 36 #include <sys/privregs.h> 37 #include <sys/psm_common.h> 38 39 /* Function prototypes of local apic and X2APIC */ 40 static uint64_t local_apic_read(uint32_t reg); 41 static void local_apic_write(uint32_t reg, uint64_t value); 42 static int get_local_apic_pri(void); 43 static void local_apic_write_task_reg(uint64_t value); 44 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1); 45 static uint64_t local_x2apic_read(uint32_t msr); 46 static void local_x2apic_write(uint32_t msr, uint64_t value); 47 static int get_local_x2apic_pri(void); 48 static void local_x2apic_write_task_reg(uint64_t value); 49 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1); 50 51 /* 52 * According to the X2APIC specification: 53 * 54 * xAPIC global enable X2APIC enable Description 55 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10]) 56 * ----------------------------------------------------------- 57 * 0 0 APIC is disabled 58 * 0 1 Invalid 59 * 1 0 APIC is enabled in xAPIC mode 60 * 1 1 APIC is enabled in X2APIC mode 61 * ----------------------------------------------------------- 62 */ 63 int x2apic_enable = 1; 64 int apic_mode = LOCAL_APIC; /* Default mode is Local APIC */ 65 66 /* Uses MMIO (Memory Mapped IO) */ 67 static apic_reg_ops_t local_apic_regs_ops = { 68 local_apic_read, 69 local_apic_write, 70 get_local_apic_pri, 71 local_apic_write_task_reg, 72 local_apic_write_int_cmd, 73 apic_send_EOI, 74 }; 75 76 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */ 77 static apic_reg_ops_t x2apic_regs_ops = { 78 local_x2apic_read, 79 local_x2apic_write, 80 get_local_x2apic_pri, 81 local_x2apic_write_task_reg, 82 local_x2apic_write_int_cmd, 83 apic_send_EOI, 84 }; 85 86 int apic_have_32bit_cr8 = 0; 87 88 /* The default ops is local APIC (Memory Mapped IO) */ 89 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops; 90 91 /* 92 * APIC register ops related data sturctures and functions. 93 */ 94 void apic_send_EOI(); 95 void apic_send_directed_EOI(uint32_t irq); 96 97 #define X2APIC_CPUID_BIT 21 98 #define X2APIC_ENABLE_BIT 10 99 100 /* 101 * Local APIC Implementation 102 */ 103 static uint64_t 104 local_apic_read(uint32_t reg) 105 { 106 return ((uint32_t)apicadr[reg]); 107 } 108 109 static void 110 local_apic_write(uint32_t reg, uint64_t value) 111 { 112 apicadr[reg] = (uint32_t)value; 113 } 114 115 static int 116 get_local_apic_pri(void) 117 { 118 #if defined(__amd64) 119 return ((int)getcr8()); 120 #else 121 if (apic_have_32bit_cr8) 122 return ((int)getcr8()); 123 return (apicadr[APIC_TASK_REG]); 124 #endif 125 } 126 127 static void 128 local_apic_write_task_reg(uint64_t value) 129 { 130 #if defined(__amd64) 131 setcr8((ulong_t)(value >> APIC_IPL_SHIFT)); 132 #else 133 if (apic_have_32bit_cr8) 134 setcr8((ulong_t)(value >> APIC_IPL_SHIFT)); 135 else 136 apicadr[APIC_TASK_REG] = (uint32_t)value; 137 #endif 138 } 139 140 static void 141 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1) 142 { 143 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET; 144 apicadr[APIC_INT_CMD1] = cmd1; 145 } 146 147 /* 148 * X2APIC Implementation. 149 */ 150 static uint64_t 151 local_x2apic_read(uint32_t msr) 152 { 153 uint64_t i; 154 155 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff); 156 return (i); 157 } 158 159 static void 160 local_x2apic_write(uint32_t msr, uint64_t value) 161 { 162 uint64_t tmp; 163 164 if (msr != APIC_EOI_REG) { 165 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)); 166 tmp = (tmp & 0xffffffff00000000) | value; 167 } else { 168 tmp = 0; 169 } 170 171 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp); 172 } 173 174 static int 175 get_local_x2apic_pri(void) 176 { 177 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2))); 178 } 179 180 static void 181 local_x2apic_write_task_reg(uint64_t value) 182 { 183 X2APIC_WRITE(APIC_TASK_REG, value); 184 } 185 186 static void 187 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1) 188 { 189 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)), 190 (((uint64_t)cpu_id << 32) | cmd1)); 191 } 192 193 /*ARGSUSED*/ 194 void 195 apic_send_EOI(uint32_t irq) 196 { 197 apic_reg_ops->apic_write(APIC_EOI_REG, 0); 198 } 199 200 /* 201 * Support for Directed EOI capability is available in both the xAPIC 202 * and x2APIC mode. 203 */ 204 void 205 apic_send_directed_EOI(uint32_t irq) 206 { 207 uchar_t ioapicindex; 208 uchar_t vector; 209 apic_irq_t *apic_irq; 210 short intr_index; 211 212 /* 213 * Following the EOI to the local APIC unit, perform a directed 214 * EOI to the IOxAPIC generating the interrupt by writing to its 215 * EOI register. 216 * 217 * A broadcast EOI is not generated. 218 */ 219 apic_reg_ops->apic_write(APIC_EOI_REG, 0); 220 221 apic_irq = apic_irq_table[irq]; 222 while (apic_irq) { 223 intr_index = apic_irq->airq_mps_intr_index; 224 if (intr_index == ACPI_INDEX || intr_index >= 0) { 225 ioapicindex = apic_irq->airq_ioapicindex; 226 vector = apic_irq->airq_vector; 227 ioapic_write_eoi(ioapicindex, vector); 228 } 229 apic_irq = apic_irq->airq_next; 230 } 231 } 232 233 int 234 apic_detect_x2apic(void) 235 { 236 struct cpuid_regs cp; 237 238 if (x2apic_enable == 0) 239 return (0); 240 241 cp.cp_eax = 1; 242 (void) __cpuid_insn(&cp); 243 244 return ((cp.cp_ecx & (0x1 << X2APIC_CPUID_BIT)) ? 1 : 0); 245 } 246 247 void 248 apic_enable_x2apic(void) 249 { 250 uint64_t apic_base_msr; 251 252 if (apic_local_mode() == LOCAL_X2APIC) { 253 /* BIOS apparently has enabled X2APIC */ 254 if (apic_mode != LOCAL_X2APIC) 255 x2apic_update_psm(); 256 return; 257 } 258 259 /* 260 * This is the first time we are enabling X2APIC on this CPU 261 */ 262 apic_base_msr = rdmsr(REG_APIC_BASE_MSR); 263 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT); 264 wrmsr(REG_APIC_BASE_MSR, apic_base_msr); 265 266 if (apic_mode != LOCAL_X2APIC) 267 x2apic_update_psm(); 268 } 269 270 /* 271 * Determine which mode the current CPU is in. See the table above. 272 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10]) 273 */ 274 int 275 apic_local_mode(void) 276 { 277 uint64_t apic_base_msr; 278 int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) | 279 (0x1 << X2APIC_ENABLE_BIT)); 280 281 apic_base_msr = rdmsr(REG_APIC_BASE_MSR); 282 283 if ((apic_base_msr & bit) == bit) 284 return (LOCAL_X2APIC); 285 else 286 return (LOCAL_APIC); 287 } 288 289 void 290 apic_set_directed_EOI_handler() 291 { 292 apic_reg_ops->apic_send_eoi = apic_send_directed_EOI; 293 } 294 295 int 296 apic_directed_EOI_supported() 297 { 298 uint32_t ver; 299 300 ver = apic_reg_ops->apic_read(APIC_VERS_REG); 301 if (ver & APIC_DIRECTED_EOI_BIT) 302 return (1); 303 304 return (0); 305 } 306 307 /* 308 * Change apic_reg_ops depending upon the apic_mode. 309 */ 310 void 311 apic_change_ops() 312 { 313 if (apic_mode == LOCAL_APIC) 314 apic_reg_ops = &local_apic_regs_ops; 315 else if (apic_mode == LOCAL_X2APIC) 316 apic_reg_ops = &x2apic_regs_ops; 317 } 318 319 /* 320 * Generates an interprocessor interrupt to another CPU when X2APIC mode is 321 * enabled. 322 */ 323 void 324 x2apic_send_ipi(int cpun, int ipl) 325 { 326 int vector; 327 ulong_t flag; 328 329 ASSERT(apic_mode == LOCAL_X2APIC); 330 331 /* 332 * With X2APIC, Intel relaxed the semantics of the 333 * WRMSR instruction such that references to the X2APIC 334 * MSR registers are no longer serializing instructions. 335 * The code that initiates IPIs assumes that some sort 336 * of memory serialization occurs. The old APIC code 337 * did a write to uncachable memory mapped registers. 338 * Any reference to uncached memory is a serializing 339 * operation. To mimic those semantics here, we do an 340 * atomic operation, which translates to a LOCK OR instruction, 341 * which is serializing. 342 */ 343 atomic_or_ulong(&flag, 1); 344 345 vector = apic_resv_vector[ipl]; 346 347 flag = intr_clear(); 348 349 /* 350 * According to X2APIC specification in section '2.3.5.1' of 351 * Interrupt Command Register Semantics, the semantics of 352 * programming Interrupt Command Register to dispatch an interrupt 353 * is simplified. A single MSR write to the 64-bit ICR is required 354 * for dispatching an interrupt. Specifically with the 64-bit MSR 355 * interface to ICR, system software is not required to check the 356 * status of the delivery status bit prior to writing to the ICR 357 * to send an IPI. With the removal of the Delivery Status bit, 358 * system software no longer has a reason to read the ICR. It remains 359 * readable only to aid in debugging. 360 */ 361 #ifdef DEBUG 362 APIC_AV_PENDING_SET(); 363 #endif /* DEBUG */ 364 365 if ((cpun == psm_get_cpu_id())) { 366 X2APIC_WRITE(X2APIC_SELF_IPI, vector); 367 } else { 368 apic_reg_ops->apic_write_int_cmd( 369 apic_cpus[cpun].aci_local_id, vector); 370 } 371 372 intr_restore(flag); 373 } 374 375 /* 376 * Generates IPI to another CPU depending on the local APIC mode. 377 * apic_send_ipi() and x2apic_send_ipi() depends on the configured 378 * mode of the local APIC, but that may not match the actual mode 379 * early in CPU startup. 380 * 381 * Any changes made to this routine must be accompanied by similar 382 * changes to apic_send_ipi(). 383 */ 384 void 385 apic_common_send_ipi(int cpun, int ipl) 386 { 387 int vector; 388 ulong_t flag; 389 int mode = apic_local_mode(); 390 391 if (mode == LOCAL_X2APIC) { 392 x2apic_send_ipi(cpun, ipl); 393 return; 394 } 395 396 ASSERT(mode == LOCAL_APIC); 397 398 vector = apic_resv_vector[ipl]; 399 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR)); 400 flag = intr_clear(); 401 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING) 402 apic_ret(); 403 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id, 404 vector); 405 intr_restore(flag); 406 } 407