1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net> 27 */ 28 29 #include <sys/cpuvar.h> 30 #include <sys/psm.h> 31 #include <sys/archsystm.h> 32 #include <sys/apic.h> 33 #include <sys/sunddi.h> 34 #include <sys/ddi_impldefs.h> 35 #include <sys/mach_intr.h> 36 #include <sys/sysmacros.h> 37 #include <sys/trap.h> 38 #include <sys/x86_archext.h> 39 #include <sys/privregs.h> 40 #include <sys/psm_common.h> 41 42 /* Function prototypes of local apic and X2APIC */ 43 static uint64_t local_apic_read(uint32_t reg); 44 static void local_apic_write(uint32_t reg, uint64_t value); 45 static int get_local_apic_pri(void); 46 static void local_apic_write_task_reg(uint64_t value); 47 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1); 48 static uint64_t local_x2apic_read(uint32_t msr); 49 static void local_x2apic_write(uint32_t msr, uint64_t value); 50 static int get_local_x2apic_pri(void); 51 static void local_x2apic_write_task_reg(uint64_t value); 52 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1); 53 54 /* 55 * According to the X2APIC specification: 56 * 57 * xAPIC global enable X2APIC enable Description 58 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10]) 59 * ----------------------------------------------------------- 60 * 0 0 APIC is disabled 61 * 0 1 Invalid 62 * 1 0 APIC is enabled in xAPIC mode 63 * 1 1 APIC is enabled in X2APIC mode 64 * ----------------------------------------------------------- 65 */ 66 int x2apic_enable = 1; 67 apic_mode_t apic_mode = LOCAL_APIC; /* Default mode is Local APIC */ 68 69 /* Uses MMIO (Memory Mapped IO) */ 70 static apic_reg_ops_t local_apic_regs_ops = { 71 local_apic_read, 72 local_apic_write, 73 get_local_apic_pri, 74 local_apic_write_task_reg, 75 local_apic_write_int_cmd, 76 apic_send_EOI, 77 }; 78 79 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */ 80 static apic_reg_ops_t x2apic_regs_ops = { 81 local_x2apic_read, 82 local_x2apic_write, 83 get_local_x2apic_pri, 84 local_x2apic_write_task_reg, 85 local_x2apic_write_int_cmd, 86 apic_send_EOI, 87 }; 88 89 int apic_have_32bit_cr8 = 0; 90 91 /* The default ops is local APIC (Memory Mapped IO) */ 92 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops; 93 94 /* 95 * APIC register ops related data sturctures and functions. 96 */ 97 void apic_send_EOI(); 98 void apic_send_directed_EOI(uint32_t irq); 99 100 #define X2APIC_ENABLE_BIT 10 101 102 /* 103 * Local APIC Implementation 104 */ 105 static uint64_t 106 local_apic_read(uint32_t reg) 107 { 108 return ((uint32_t)apicadr[reg]); 109 } 110 111 static void 112 local_apic_write(uint32_t reg, uint64_t value) 113 { 114 apicadr[reg] = (uint32_t)value; 115 } 116 117 static int 118 get_local_apic_pri(void) 119 { 120 #if defined(__amd64) 121 return ((int)getcr8()); 122 #else 123 if (apic_have_32bit_cr8) 124 return ((int)getcr8()); 125 return (apicadr[APIC_TASK_REG]); 126 #endif 127 } 128 129 static void 130 local_apic_write_task_reg(uint64_t value) 131 { 132 #if defined(__amd64) 133 setcr8((ulong_t)(value >> APIC_IPL_SHIFT)); 134 #else 135 if (apic_have_32bit_cr8) 136 setcr8((ulong_t)(value >> APIC_IPL_SHIFT)); 137 else 138 apicadr[APIC_TASK_REG] = (uint32_t)value; 139 #endif 140 } 141 142 static void 143 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1) 144 { 145 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET; 146 apicadr[APIC_INT_CMD1] = cmd1; 147 } 148 149 /* 150 * X2APIC Implementation. 151 */ 152 static uint64_t 153 local_x2apic_read(uint32_t msr) 154 { 155 uint64_t i; 156 157 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff); 158 return (i); 159 } 160 161 static void 162 local_x2apic_write(uint32_t msr, uint64_t value) 163 { 164 uint64_t tmp; 165 166 if (msr != APIC_EOI_REG) { 167 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)); 168 tmp = (tmp & 0xffffffff00000000) | value; 169 } else { 170 tmp = 0; 171 } 172 173 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp); 174 } 175 176 static int 177 get_local_x2apic_pri(void) 178 { 179 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2))); 180 } 181 182 static void 183 local_x2apic_write_task_reg(uint64_t value) 184 { 185 X2APIC_WRITE(APIC_TASK_REG, value); 186 } 187 188 static void 189 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1) 190 { 191 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)), 192 (((uint64_t)cpu_id << 32) | cmd1)); 193 } 194 195 /*ARGSUSED*/ 196 void 197 apic_send_EOI(uint32_t irq) 198 { 199 apic_reg_ops->apic_write(APIC_EOI_REG, 0); 200 } 201 202 /* 203 * Support for Directed EOI capability is available in both the xAPIC 204 * and x2APIC mode. 205 */ 206 void 207 apic_send_directed_EOI(uint32_t irq) 208 { 209 uchar_t ioapicindex; 210 uchar_t vector; 211 apic_irq_t *apic_irq; 212 short intr_index; 213 214 /* 215 * Following the EOI to the local APIC unit, perform a directed 216 * EOI to the IOxAPIC generating the interrupt by writing to its 217 * EOI register. 218 * 219 * A broadcast EOI is not generated. 220 */ 221 apic_reg_ops->apic_write(APIC_EOI_REG, 0); 222 223 apic_irq = apic_irq_table[irq]; 224 while (apic_irq) { 225 intr_index = apic_irq->airq_mps_intr_index; 226 if (intr_index == ACPI_INDEX || intr_index >= 0) { 227 ioapicindex = apic_irq->airq_ioapicindex; 228 vector = apic_irq->airq_vector; 229 ioapic_write_eoi(ioapicindex, vector); 230 } 231 apic_irq = apic_irq->airq_next; 232 } 233 } 234 235 int 236 apic_detect_x2apic(void) 237 { 238 if (x2apic_enable == 0) 239 return (0); 240 241 return (is_x86_feature(x86_featureset, X86FSET_X2APIC)); 242 } 243 244 void 245 apic_enable_x2apic(void) 246 { 247 uint64_t apic_base_msr; 248 249 if (apic_local_mode() == LOCAL_X2APIC) { 250 /* BIOS apparently has enabled X2APIC */ 251 if (apic_mode != LOCAL_X2APIC) 252 x2apic_update_psm(); 253 return; 254 } 255 256 /* 257 * This is the first time we are enabling X2APIC on this CPU 258 */ 259 apic_base_msr = rdmsr(REG_APIC_BASE_MSR); 260 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT); 261 wrmsr(REG_APIC_BASE_MSR, apic_base_msr); 262 263 if (apic_mode != LOCAL_X2APIC) 264 x2apic_update_psm(); 265 } 266 267 /* 268 * Determine which mode the current CPU is in. See the table above. 269 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10]) 270 */ 271 int 272 apic_local_mode(void) 273 { 274 uint64_t apic_base_msr; 275 int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) | 276 (0x1 << X2APIC_ENABLE_BIT)); 277 278 apic_base_msr = rdmsr(REG_APIC_BASE_MSR); 279 280 if ((apic_base_msr & bit) == bit) 281 return (LOCAL_X2APIC); 282 else 283 return (LOCAL_APIC); 284 } 285 286 void 287 apic_set_directed_EOI_handler() 288 { 289 apic_reg_ops->apic_send_eoi = apic_send_directed_EOI; 290 } 291 292 int 293 apic_directed_EOI_supported() 294 { 295 uint32_t ver; 296 297 ver = apic_reg_ops->apic_read(APIC_VERS_REG); 298 if (ver & APIC_DIRECTED_EOI_BIT) 299 return (1); 300 301 return (0); 302 } 303 304 /* 305 * Change apic_reg_ops depending upon the apic_mode. 306 */ 307 void 308 apic_change_ops() 309 { 310 if (apic_mode == LOCAL_APIC) 311 apic_reg_ops = &local_apic_regs_ops; 312 else if (apic_mode == LOCAL_X2APIC) 313 apic_reg_ops = &x2apic_regs_ops; 314 } 315 316 /* 317 * Generates an interprocessor interrupt to another CPU when X2APIC mode is 318 * enabled. 319 */ 320 void 321 x2apic_send_ipi(int cpun, int ipl) 322 { 323 int vector; 324 ulong_t flag; 325 326 ASSERT(apic_mode == LOCAL_X2APIC); 327 328 /* 329 * With X2APIC, Intel relaxed the semantics of the 330 * WRMSR instruction such that references to the X2APIC 331 * MSR registers are no longer serializing instructions. 332 * The code that initiates IPIs assumes that some sort 333 * of memory serialization occurs. The old APIC code 334 * did a write to uncachable memory mapped registers. 335 * Any reference to uncached memory is a serializing 336 * operation. To mimic those semantics here, we do an 337 * atomic operation, which translates to a LOCK OR instruction, 338 * which is serializing. 339 */ 340 atomic_or_ulong(&flag, 1); 341 342 vector = apic_resv_vector[ipl]; 343 344 flag = intr_clear(); 345 346 /* 347 * According to X2APIC specification in section '2.3.5.1' of 348 * Interrupt Command Register Semantics, the semantics of 349 * programming Interrupt Command Register to dispatch an interrupt 350 * is simplified. A single MSR write to the 64-bit ICR is required 351 * for dispatching an interrupt. Specifically with the 64-bit MSR 352 * interface to ICR, system software is not required to check the 353 * status of the delivery status bit prior to writing to the ICR 354 * to send an IPI. With the removal of the Delivery Status bit, 355 * system software no longer has a reason to read the ICR. It remains 356 * readable only to aid in debugging. 357 */ 358 #ifdef DEBUG 359 APIC_AV_PENDING_SET(); 360 #endif /* DEBUG */ 361 362 if ((cpun == psm_get_cpu_id())) { 363 X2APIC_WRITE(X2APIC_SELF_IPI, vector); 364 } else { 365 apic_reg_ops->apic_write_int_cmd( 366 apic_cpus[cpun].aci_local_id, vector); 367 } 368 369 intr_restore(flag); 370 } 371 372 /* 373 * Generates IPI to another CPU depending on the local APIC mode. 374 * apic_send_ipi() and x2apic_send_ipi() depends on the configured 375 * mode of the local APIC, but that may not match the actual mode 376 * early in CPU startup. 377 * 378 * Any changes made to this routine must be accompanied by similar 379 * changes to apic_send_ipi(). 380 */ 381 void 382 apic_common_send_ipi(int cpun, int ipl) 383 { 384 int vector; 385 ulong_t flag; 386 int mode = apic_local_mode(); 387 388 if (mode == LOCAL_X2APIC) { 389 x2apic_send_ipi(cpun, ipl); 390 return; 391 } 392 393 ASSERT(mode == LOCAL_APIC); 394 395 vector = apic_resv_vector[ipl]; 396 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR)); 397 flag = intr_clear(); 398 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING) 399 apic_ret(); 400 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id, 401 vector); 402 intr_restore(flag); 403 } 404