1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 /* 26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net> 27 * Copyright (c) 2014 by Delphix. All rights reserved. 28 * Copyright 2018 Joyent, Inc. 29 */ 30 31 #include <sys/cpuvar.h> 32 #include <sys/psm.h> 33 #include <sys/archsystm.h> 34 #include <sys/apic.h> 35 #include <sys/apic_common.h> 36 #include <sys/sunddi.h> 37 #include <sys/ddi_impldefs.h> 38 #include <sys/mach_intr.h> 39 #include <sys/sysmacros.h> 40 #include <sys/trap.h> 41 #include <sys/x86_archext.h> 42 #include <sys/privregs.h> 43 #include <sys/psm_common.h> 44 45 /* Function prototypes of X2APIC */ 46 static uint64_t local_x2apic_read(uint32_t msr); 47 static void local_x2apic_write(uint32_t msr, uint64_t value); 48 static int get_local_x2apic_pri(void); 49 static void local_x2apic_write_task_reg(uint64_t value); 50 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1); 51 52 /* 53 * According to the X2APIC specification: 54 * 55 * xAPIC global enable X2APIC enable Description 56 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10]) 57 * ----------------------------------------------------------- 58 * 0 0 APIC is disabled 59 * 0 1 Invalid 60 * 1 0 APIC is enabled in xAPIC mode 61 * 1 1 APIC is enabled in X2APIC mode 62 * ----------------------------------------------------------- 63 */ 64 int x2apic_enable = 1; 65 66 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */ 67 static apic_reg_ops_t x2apic_regs_ops = { 68 local_x2apic_read, 69 local_x2apic_write, 70 get_local_x2apic_pri, 71 local_x2apic_write_task_reg, 72 local_x2apic_write_int_cmd, 73 apic_send_EOI, 74 }; 75 76 /* 77 * X2APIC Implementation. 78 */ 79 static uint64_t 80 local_x2apic_read(uint32_t msr) 81 { 82 uint64_t i; 83 84 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff); 85 return (i); 86 } 87 88 static void 89 local_x2apic_write(uint32_t msr, uint64_t value) 90 { 91 uint64_t tmp; 92 93 if (msr != APIC_EOI_REG) { 94 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)); 95 tmp = (tmp & 0xffffffff00000000) | value; 96 } else { 97 tmp = 0; 98 } 99 100 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp); 101 } 102 103 static int 104 get_local_x2apic_pri(void) 105 { 106 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2))); 107 } 108 109 static void 110 local_x2apic_write_task_reg(uint64_t value) 111 { 112 X2APIC_WRITE(APIC_TASK_REG, value); 113 } 114 115 static void 116 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1) 117 { 118 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)), 119 (((uint64_t)cpu_id << 32) | cmd1)); 120 } 121 122 int 123 apic_detect_x2apic(void) 124 { 125 if (x2apic_enable == 0) 126 return (0); 127 128 return (is_x86_feature(x86_featureset, X86FSET_X2APIC)); 129 } 130 131 void 132 apic_enable_x2apic(void) 133 { 134 uint64_t apic_base_msr; 135 136 if (apic_local_mode() == LOCAL_X2APIC) { 137 /* BIOS apparently has enabled X2APIC */ 138 if (apic_mode != LOCAL_X2APIC) 139 x2apic_update_psm(); 140 return; 141 } 142 143 /* 144 * This is the first time we are enabling X2APIC on this CPU 145 */ 146 apic_base_msr = rdmsr(REG_APIC_BASE_MSR); 147 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT); 148 wrmsr(REG_APIC_BASE_MSR, apic_base_msr); 149 150 if (apic_mode != LOCAL_X2APIC) 151 x2apic_update_psm(); 152 } 153 154 /* 155 * Change apic_reg_ops depending upon the apic_mode. 156 */ 157 void 158 apic_change_ops() 159 { 160 if (apic_mode == LOCAL_APIC) 161 apic_reg_ops = &local_apic_regs_ops; 162 else if (apic_mode == LOCAL_X2APIC) 163 apic_reg_ops = &x2apic_regs_ops; 164 } 165 166 /* 167 * Generates an interprocessor interrupt to another CPU when X2APIC mode is 168 * enabled. 169 */ 170 void 171 x2apic_send_ipi(int cpun, int ipl) 172 { 173 int vector; 174 ulong_t flag; 175 176 ASSERT(apic_mode == LOCAL_X2APIC); 177 178 /* 179 * With X2APIC, Intel relaxed the semantics of the 180 * WRMSR instruction such that references to the X2APIC 181 * MSR registers are no longer serializing instructions. 182 * The code that initiates IPIs assumes that some sort 183 * of memory serialization occurs. The old APIC code 184 * did a write to uncachable memory mapped registers. 185 * Any reference to uncached memory is a serializing 186 * operation. To mimic those semantics here, we do an 187 * atomic operation, which translates to a LOCK OR instruction, 188 * which is serializing. 189 */ 190 atomic_or_ulong(&flag, 1); 191 192 vector = apic_resv_vector[ipl]; 193 194 flag = intr_clear(); 195 196 /* 197 * According to X2APIC specification in section '2.3.5.1' of 198 * Interrupt Command Register Semantics, the semantics of 199 * programming Interrupt Command Register to dispatch an interrupt 200 * is simplified. A single MSR write to the 64-bit ICR is required 201 * for dispatching an interrupt. Specifically with the 64-bit MSR 202 * interface to ICR, system software is not required to check the 203 * status of the delivery status bit prior to writing to the ICR 204 * to send an IPI. With the removal of the Delivery Status bit, 205 * system software no longer has a reason to read the ICR. It remains 206 * readable only to aid in debugging. 207 */ 208 #ifdef DEBUG 209 APIC_AV_PENDING_SET(); 210 #endif /* DEBUG */ 211 212 if ((cpun == psm_get_cpu_id())) { 213 X2APIC_WRITE(X2APIC_SELF_IPI, vector); 214 } else { 215 apic_reg_ops->apic_write_int_cmd( 216 apic_cpus[cpun].aci_local_id, vector); 217 } 218 219 intr_restore(flag); 220 } 221 222 void 223 x2apic_send_pir_ipi(processorid_t cpun) 224 { 225 const int vector = apic_pir_vect; 226 ulong_t flag; 227 228 ASSERT(apic_mode == LOCAL_X2APIC); 229 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR)); 230 231 /* Serialize as described in x2apic_send_ipi() above. */ 232 atomic_or_ulong(&flag, 1); 233 234 flag = intr_clear(); 235 236 /* Self-IPI for inducing PIR makes no sense. */ 237 if ((cpun != psm_get_cpu_id())) { 238 #ifdef DEBUG 239 /* Only for debugging. (again, see: x2apic_send_ipi) */ 240 APIC_AV_PENDING_SET(); 241 #endif /* DEBUG */ 242 243 apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id, 244 vector); 245 } 246 247 intr_restore(flag); 248 } 249 250 /* 251 * Generates IPI to another CPU depending on the local APIC mode. 252 * apic_send_ipi() and x2apic_send_ipi() depends on the configured 253 * mode of the local APIC, but that may not match the actual mode 254 * early in CPU startup. 255 * 256 * Any changes made to this routine must be accompanied by similar 257 * changes to apic_send_ipi(). 258 */ 259 void 260 apic_common_send_ipi(int cpun, int ipl) 261 { 262 int vector; 263 ulong_t flag; 264 int mode = apic_local_mode(); 265 266 if (mode == LOCAL_X2APIC) { 267 x2apic_send_ipi(cpun, ipl); 268 return; 269 } 270 271 ASSERT(mode == LOCAL_APIC); 272 273 vector = apic_resv_vector[ipl]; 274 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR)); 275 flag = intr_clear(); 276 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING) 277 apic_ret(); 278 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id, 279 vector); 280 intr_restore(flag); 281 } 282 283 void 284 apic_common_send_pir_ipi(processorid_t cpun) 285 { 286 const int mode = apic_local_mode(); 287 288 if (mode == LOCAL_X2APIC) { 289 x2apic_send_pir_ipi(cpun); 290 return; 291 } 292 293 apic_send_pir_ipi(cpun); 294 } 295