1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net> 27 * Copyright (c) 2014 by Delphix. All rights reserved. 28 */ 29 30 #include <sys/cpuvar.h> 31 #include <sys/psm.h> 32 #include <sys/archsystm.h> 33 #include <sys/apic.h> 34 #include <sys/sunddi.h> 35 #include <sys/ddi_impldefs.h> 36 #include <sys/mach_intr.h> 37 #include <sys/sysmacros.h> 38 #include <sys/trap.h> 39 #include <sys/x86_archext.h> 40 #include <sys/privregs.h> 41 #include <sys/psm_common.h> 42 43 /* Function prototypes of local apic and X2APIC */ 44 static uint64_t local_apic_read(uint32_t reg); 45 static void local_apic_write(uint32_t reg, uint64_t value); 46 static int get_local_apic_pri(void); 47 static void local_apic_write_task_reg(uint64_t value); 48 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1); 49 static uint64_t local_x2apic_read(uint32_t msr); 50 static void local_x2apic_write(uint32_t msr, uint64_t value); 51 static int get_local_x2apic_pri(void); 52 static void local_x2apic_write_task_reg(uint64_t value); 53 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1); 54 55 /* 56 * According to the X2APIC specification: 57 * 58 * xAPIC global enable X2APIC enable Description 59 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10]) 60 * ----------------------------------------------------------- 61 * 0 0 APIC is disabled 62 * 0 1 Invalid 63 * 1 0 APIC is enabled in xAPIC mode 64 * 1 1 APIC is enabled in X2APIC mode 65 * ----------------------------------------------------------- 66 */ 67 int x2apic_enable = 1; 68 apic_mode_t apic_mode = LOCAL_APIC; /* Default mode is Local APIC */ 69 70 /* See apic_directed_EOI_supported(). Currently 3-state variable. */ 71 volatile int apic_directed_eoi_state = 2; 72 73 /* Uses MMIO (Memory Mapped IO) */ 74 static apic_reg_ops_t local_apic_regs_ops = { 75 local_apic_read, 76 local_apic_write, 77 get_local_apic_pri, 78 local_apic_write_task_reg, 79 local_apic_write_int_cmd, 80 apic_send_EOI, 81 }; 82 83 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */ 84 static apic_reg_ops_t x2apic_regs_ops = { 85 local_x2apic_read, 86 local_x2apic_write, 87 get_local_x2apic_pri, 88 local_x2apic_write_task_reg, 89 local_x2apic_write_int_cmd, 90 apic_send_EOI, 91 }; 92 93 int apic_have_32bit_cr8 = 0; 94 95 /* The default ops is local APIC (Memory Mapped IO) */ 96 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops; 97 98 /* 99 * APIC register ops related data sturctures and functions. 100 */ 101 void apic_send_EOI(); 102 void apic_send_directed_EOI(uint32_t irq); 103 104 #define X2APIC_ENABLE_BIT 10 105 106 /* 107 * Local APIC Implementation 108 */ 109 static uint64_t 110 local_apic_read(uint32_t reg) 111 { 112 return ((uint32_t)apicadr[reg]); 113 } 114 115 static void 116 local_apic_write(uint32_t reg, uint64_t value) 117 { 118 apicadr[reg] = (uint32_t)value; 119 } 120 121 static int 122 get_local_apic_pri(void) 123 { 124 #if defined(__amd64) 125 return ((int)getcr8()); 126 #else 127 if (apic_have_32bit_cr8) 128 return ((int)getcr8()); 129 return (apicadr[APIC_TASK_REG]); 130 #endif 131 } 132 133 static void 134 local_apic_write_task_reg(uint64_t value) 135 { 136 #if defined(__amd64) 137 setcr8((ulong_t)(value >> APIC_IPL_SHIFT)); 138 #else 139 if (apic_have_32bit_cr8) 140 setcr8((ulong_t)(value >> APIC_IPL_SHIFT)); 141 else 142 apicadr[APIC_TASK_REG] = (uint32_t)value; 143 #endif 144 } 145 146 static void 147 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1) 148 { 149 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET; 150 apicadr[APIC_INT_CMD1] = cmd1; 151 } 152 153 /* 154 * X2APIC Implementation. 155 */ 156 static uint64_t 157 local_x2apic_read(uint32_t msr) 158 { 159 uint64_t i; 160 161 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff); 162 return (i); 163 } 164 165 static void 166 local_x2apic_write(uint32_t msr, uint64_t value) 167 { 168 uint64_t tmp; 169 170 if (msr != APIC_EOI_REG) { 171 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)); 172 tmp = (tmp & 0xffffffff00000000) | value; 173 } else { 174 tmp = 0; 175 } 176 177 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp); 178 } 179 180 static int 181 get_local_x2apic_pri(void) 182 { 183 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2))); 184 } 185 186 static void 187 local_x2apic_write_task_reg(uint64_t value) 188 { 189 X2APIC_WRITE(APIC_TASK_REG, value); 190 } 191 192 static void 193 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1) 194 { 195 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)), 196 (((uint64_t)cpu_id << 32) | cmd1)); 197 } 198 199 /*ARGSUSED*/ 200 void 201 apic_send_EOI(uint32_t irq) 202 { 203 apic_reg_ops->apic_write(APIC_EOI_REG, 0); 204 } 205 206 /* 207 * Support for Directed EOI capability is available in both the xAPIC 208 * and x2APIC mode. 209 */ 210 void 211 apic_send_directed_EOI(uint32_t irq) 212 { 213 uchar_t ioapicindex; 214 uchar_t vector; 215 apic_irq_t *apic_irq; 216 short intr_index; 217 218 /* 219 * Following the EOI to the local APIC unit, perform a directed 220 * EOI to the IOxAPIC generating the interrupt by writing to its 221 * EOI register. 222 * 223 * A broadcast EOI is not generated. 224 */ 225 apic_reg_ops->apic_write(APIC_EOI_REG, 0); 226 227 apic_irq = apic_irq_table[irq]; 228 while (apic_irq) { 229 intr_index = apic_irq->airq_mps_intr_index; 230 if (intr_index == ACPI_INDEX || intr_index >= 0) { 231 ioapicindex = apic_irq->airq_ioapicindex; 232 vector = apic_irq->airq_vector; 233 ioapic_write_eoi(ioapicindex, vector); 234 } 235 apic_irq = apic_irq->airq_next; 236 } 237 } 238 239 int 240 apic_detect_x2apic(void) 241 { 242 if (x2apic_enable == 0) 243 return (0); 244 245 return (is_x86_feature(x86_featureset, X86FSET_X2APIC)); 246 } 247 248 void 249 apic_enable_x2apic(void) 250 { 251 uint64_t apic_base_msr; 252 253 if (apic_local_mode() == LOCAL_X2APIC) { 254 /* BIOS apparently has enabled X2APIC */ 255 if (apic_mode != LOCAL_X2APIC) 256 x2apic_update_psm(); 257 return; 258 } 259 260 /* 261 * This is the first time we are enabling X2APIC on this CPU 262 */ 263 apic_base_msr = rdmsr(REG_APIC_BASE_MSR); 264 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT); 265 wrmsr(REG_APIC_BASE_MSR, apic_base_msr); 266 267 if (apic_mode != LOCAL_X2APIC) 268 x2apic_update_psm(); 269 } 270 271 /* 272 * Determine which mode the current CPU is in. See the table above. 273 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10]) 274 */ 275 int 276 apic_local_mode(void) 277 { 278 uint64_t apic_base_msr; 279 int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) | 280 (0x1 << X2APIC_ENABLE_BIT)); 281 282 apic_base_msr = rdmsr(REG_APIC_BASE_MSR); 283 284 if ((apic_base_msr & bit) == bit) 285 return (LOCAL_X2APIC); 286 else 287 return (LOCAL_APIC); 288 } 289 290 void 291 apic_set_directed_EOI_handler() 292 { 293 apic_reg_ops->apic_send_eoi = apic_send_directed_EOI; 294 } 295 296 int 297 apic_directed_EOI_supported() 298 { 299 uint32_t ver; 300 301 /* 302 * There are some known issues with some versions of Linux KVM and QEMU 303 * where by directed EOIs do not properly function and instead get 304 * coalesced at the hypervisor, causing the host not to see interrupts. 305 * Thus, when the platform is KVM, we would like to disable it by 306 * default, but keep it available otherwise. 307 * 308 * We use a three-state variable (apic_directed_eoi_state) to determine 309 * how we handle directed EOI. 310 * 311 * 0 --> Don't do directed EOI at all. 312 * 1 --> Do directed EOI if available, no matter the HW environment. 313 * 2 --> Don't do directed EOI on KVM, but do it otherwise if available. 314 * 315 * If some grinning weirdo put something else in there, treat it as '2' 316 * (i.e. the current default). 317 * 318 * Note, at this time illumos KVM does not identify as KVM. If it does, 319 * we'll need to do some work to determine if it should be caught by 320 * this or if it should show up as its own value of platform_type. 321 */ 322 switch (apic_directed_eoi_state) { 323 case 0: 324 /* Don't do it at all. */ 325 return (0); 326 case 1: 327 break; 328 case 2: 329 default: 330 /* Only do it if we aren't on KVM. */ 331 if (get_hwenv() == HW_KVM) 332 return (0); 333 /* FALLTHRU */ 334 } 335 336 ver = apic_reg_ops->apic_read(APIC_VERS_REG); 337 if (ver & APIC_DIRECTED_EOI_BIT) 338 return (1); 339 340 return (0); 341 } 342 343 /* 344 * Change apic_reg_ops depending upon the apic_mode. 345 */ 346 void 347 apic_change_ops() 348 { 349 if (apic_mode == LOCAL_APIC) 350 apic_reg_ops = &local_apic_regs_ops; 351 else if (apic_mode == LOCAL_X2APIC) 352 apic_reg_ops = &x2apic_regs_ops; 353 } 354 355 /* 356 * Generates an interprocessor interrupt to another CPU when X2APIC mode is 357 * enabled. 358 */ 359 void 360 x2apic_send_ipi(int cpun, int ipl) 361 { 362 int vector; 363 ulong_t flag; 364 365 ASSERT(apic_mode == LOCAL_X2APIC); 366 367 /* 368 * With X2APIC, Intel relaxed the semantics of the 369 * WRMSR instruction such that references to the X2APIC 370 * MSR registers are no longer serializing instructions. 371 * The code that initiates IPIs assumes that some sort 372 * of memory serialization occurs. The old APIC code 373 * did a write to uncachable memory mapped registers. 374 * Any reference to uncached memory is a serializing 375 * operation. To mimic those semantics here, we do an 376 * atomic operation, which translates to a LOCK OR instruction, 377 * which is serializing. 378 */ 379 atomic_or_ulong(&flag, 1); 380 381 vector = apic_resv_vector[ipl]; 382 383 flag = intr_clear(); 384 385 /* 386 * According to X2APIC specification in section '2.3.5.1' of 387 * Interrupt Command Register Semantics, the semantics of 388 * programming Interrupt Command Register to dispatch an interrupt 389 * is simplified. A single MSR write to the 64-bit ICR is required 390 * for dispatching an interrupt. Specifically with the 64-bit MSR 391 * interface to ICR, system software is not required to check the 392 * status of the delivery status bit prior to writing to the ICR 393 * to send an IPI. With the removal of the Delivery Status bit, 394 * system software no longer has a reason to read the ICR. It remains 395 * readable only to aid in debugging. 396 */ 397 #ifdef DEBUG 398 APIC_AV_PENDING_SET(); 399 #endif /* DEBUG */ 400 401 if ((cpun == psm_get_cpu_id())) { 402 X2APIC_WRITE(X2APIC_SELF_IPI, vector); 403 } else { 404 apic_reg_ops->apic_write_int_cmd( 405 apic_cpus[cpun].aci_local_id, vector); 406 } 407 408 intr_restore(flag); 409 } 410 411 /* 412 * Generates IPI to another CPU depending on the local APIC mode. 413 * apic_send_ipi() and x2apic_send_ipi() depends on the configured 414 * mode of the local APIC, but that may not match the actual mode 415 * early in CPU startup. 416 * 417 * Any changes made to this routine must be accompanied by similar 418 * changes to apic_send_ipi(). 419 */ 420 void 421 apic_common_send_ipi(int cpun, int ipl) 422 { 423 int vector; 424 ulong_t flag; 425 int mode = apic_local_mode(); 426 427 if (mode == LOCAL_X2APIC) { 428 x2apic_send_ipi(cpun, ipl); 429 return; 430 } 431 432 ASSERT(mode == LOCAL_APIC); 433 434 vector = apic_resv_vector[ipl]; 435 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR)); 436 flag = intr_clear(); 437 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING) 438 apic_ret(); 439 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id, 440 vector); 441 intr_restore(flag); 442 } 443