1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /*
26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 */
28
29 #include <sys/cpuvar.h>
30 #include <sys/psm.h>
31 #include <sys/archsystm.h>
32 #include <sys/apic.h>
33 #include <sys/sunddi.h>
34 #include <sys/ddi_impldefs.h>
35 #include <sys/mach_intr.h>
36 #include <sys/sysmacros.h>
37 #include <sys/trap.h>
38 #include <sys/x86_archext.h>
39 #include <sys/privregs.h>
40 #include <sys/psm_common.h>
41
42 /* Function prototypes of local apic and X2APIC */
43 static uint64_t local_apic_read(uint32_t reg);
44 static void local_apic_write(uint32_t reg, uint64_t value);
45 static int get_local_apic_pri(void);
46 static void local_apic_write_task_reg(uint64_t value);
47 static void local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
48 static uint64_t local_x2apic_read(uint32_t msr);
49 static void local_x2apic_write(uint32_t msr, uint64_t value);
50 static int get_local_x2apic_pri(void);
51 static void local_x2apic_write_task_reg(uint64_t value);
52 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
53
54 /*
55 * According to the X2APIC specification:
56 *
57 * xAPIC global enable X2APIC enable Description
58 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
59 * -----------------------------------------------------------
60 * 0 0 APIC is disabled
61 * 0 1 Invalid
62 * 1 0 APIC is enabled in xAPIC mode
63 * 1 1 APIC is enabled in X2APIC mode
64 * -----------------------------------------------------------
65 */
66 int x2apic_enable = 1;
67 apic_mode_t apic_mode = LOCAL_APIC; /* Default mode is Local APIC */
68
69 /* Uses MMIO (Memory Mapped IO) */
70 static apic_reg_ops_t local_apic_regs_ops = {
71 local_apic_read,
72 local_apic_write,
73 get_local_apic_pri,
74 local_apic_write_task_reg,
75 local_apic_write_int_cmd,
76 apic_send_EOI,
77 };
78
79 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
80 static apic_reg_ops_t x2apic_regs_ops = {
81 local_x2apic_read,
82 local_x2apic_write,
83 get_local_x2apic_pri,
84 local_x2apic_write_task_reg,
85 local_x2apic_write_int_cmd,
86 apic_send_EOI,
87 };
88
89 int apic_have_32bit_cr8 = 0;
90
91 /* The default ops is local APIC (Memory Mapped IO) */
92 apic_reg_ops_t *apic_reg_ops = &local_apic_regs_ops;
93
94 /*
95 * APIC register ops related data sturctures and functions.
96 */
97 void apic_send_EOI();
98 void apic_send_directed_EOI(uint32_t irq);
99
100 #define X2APIC_ENABLE_BIT 10
101
102 /*
103 * Local APIC Implementation
104 */
105 static uint64_t
local_apic_read(uint32_t reg)106 local_apic_read(uint32_t reg)
107 {
108 return ((uint32_t)apicadr[reg]);
109 }
110
111 static void
local_apic_write(uint32_t reg,uint64_t value)112 local_apic_write(uint32_t reg, uint64_t value)
113 {
114 apicadr[reg] = (uint32_t)value;
115 }
116
117 static int
get_local_apic_pri(void)118 get_local_apic_pri(void)
119 {
120 #if defined(__amd64)
121 return ((int)getcr8());
122 #else
123 if (apic_have_32bit_cr8)
124 return ((int)getcr8());
125 return (apicadr[APIC_TASK_REG]);
126 #endif
127 }
128
129 static void
local_apic_write_task_reg(uint64_t value)130 local_apic_write_task_reg(uint64_t value)
131 {
132 #if defined(__amd64)
133 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
134 #else
135 if (apic_have_32bit_cr8)
136 setcr8((ulong_t)(value >> APIC_IPL_SHIFT));
137 else
138 apicadr[APIC_TASK_REG] = (uint32_t)value;
139 #endif
140 }
141
142 static void
local_apic_write_int_cmd(uint32_t cpu_id,uint32_t cmd1)143 local_apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
144 {
145 apicadr[APIC_INT_CMD2] = cpu_id << APIC_ICR_ID_BIT_OFFSET;
146 apicadr[APIC_INT_CMD1] = cmd1;
147 }
148
149 /*
150 * X2APIC Implementation.
151 */
152 static uint64_t
local_x2apic_read(uint32_t msr)153 local_x2apic_read(uint32_t msr)
154 {
155 uint64_t i;
156
157 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
158 return (i);
159 }
160
161 static void
local_x2apic_write(uint32_t msr,uint64_t value)162 local_x2apic_write(uint32_t msr, uint64_t value)
163 {
164 uint64_t tmp;
165
166 if (msr != APIC_EOI_REG) {
167 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
168 tmp = (tmp & 0xffffffff00000000) | value;
169 } else {
170 tmp = 0;
171 }
172
173 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
174 }
175
176 static int
get_local_x2apic_pri(void)177 get_local_x2apic_pri(void)
178 {
179 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
180 }
181
182 static void
local_x2apic_write_task_reg(uint64_t value)183 local_x2apic_write_task_reg(uint64_t value)
184 {
185 X2APIC_WRITE(APIC_TASK_REG, value);
186 }
187
188 static void
local_x2apic_write_int_cmd(uint32_t cpu_id,uint32_t cmd1)189 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
190 {
191 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
192 (((uint64_t)cpu_id << 32) | cmd1));
193 }
194
195 /*ARGSUSED*/
196 void
apic_send_EOI(uint32_t irq)197 apic_send_EOI(uint32_t irq)
198 {
199 apic_reg_ops->apic_write(APIC_EOI_REG, 0);
200 }
201
202 /*
203 * Support for Directed EOI capability is available in both the xAPIC
204 * and x2APIC mode.
205 */
206 void
apic_send_directed_EOI(uint32_t irq)207 apic_send_directed_EOI(uint32_t irq)
208 {
209 uchar_t ioapicindex;
210 uchar_t vector;
211 apic_irq_t *apic_irq;
212 short intr_index;
213
214 /*
215 * Following the EOI to the local APIC unit, perform a directed
216 * EOI to the IOxAPIC generating the interrupt by writing to its
217 * EOI register.
218 *
219 * A broadcast EOI is not generated.
220 */
221 apic_reg_ops->apic_write(APIC_EOI_REG, 0);
222
223 apic_irq = apic_irq_table[irq];
224 while (apic_irq) {
225 intr_index = apic_irq->airq_mps_intr_index;
226 if (intr_index == ACPI_INDEX || intr_index >= 0) {
227 ioapicindex = apic_irq->airq_ioapicindex;
228 vector = apic_irq->airq_vector;
229 ioapic_write_eoi(ioapicindex, vector);
230 }
231 apic_irq = apic_irq->airq_next;
232 }
233 }
234
235 int
apic_detect_x2apic(void)236 apic_detect_x2apic(void)
237 {
238 if (x2apic_enable == 0)
239 return (0);
240
241 return (is_x86_feature(x86_featureset, X86FSET_X2APIC));
242 }
243
244 void
apic_enable_x2apic(void)245 apic_enable_x2apic(void)
246 {
247 uint64_t apic_base_msr;
248
249 if (apic_local_mode() == LOCAL_X2APIC) {
250 /* BIOS apparently has enabled X2APIC */
251 if (apic_mode != LOCAL_X2APIC)
252 x2apic_update_psm();
253 return;
254 }
255
256 /*
257 * This is the first time we are enabling X2APIC on this CPU
258 */
259 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
260 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
261 wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
262
263 if (apic_mode != LOCAL_X2APIC)
264 x2apic_update_psm();
265 }
266
267 /*
268 * Determine which mode the current CPU is in. See the table above.
269 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
270 */
271 int
apic_local_mode(void)272 apic_local_mode(void)
273 {
274 uint64_t apic_base_msr;
275 int bit = ((0x1 << (X2APIC_ENABLE_BIT + 1)) |
276 (0x1 << X2APIC_ENABLE_BIT));
277
278 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
279
280 if ((apic_base_msr & bit) == bit)
281 return (LOCAL_X2APIC);
282 else
283 return (LOCAL_APIC);
284 }
285
286 void
apic_set_directed_EOI_handler()287 apic_set_directed_EOI_handler()
288 {
289 apic_reg_ops->apic_send_eoi = apic_send_directed_EOI;
290 }
291
292 int
apic_directed_EOI_supported()293 apic_directed_EOI_supported()
294 {
295 uint32_t ver;
296
297 ver = apic_reg_ops->apic_read(APIC_VERS_REG);
298 if (ver & APIC_DIRECTED_EOI_BIT)
299 return (1);
300
301 return (0);
302 }
303
304 /*
305 * Change apic_reg_ops depending upon the apic_mode.
306 */
307 void
apic_change_ops()308 apic_change_ops()
309 {
310 if (apic_mode == LOCAL_APIC)
311 apic_reg_ops = &local_apic_regs_ops;
312 else if (apic_mode == LOCAL_X2APIC)
313 apic_reg_ops = &x2apic_regs_ops;
314 }
315
316 /*
317 * Generates an interprocessor interrupt to another CPU when X2APIC mode is
318 * enabled.
319 */
320 void
x2apic_send_ipi(int cpun,int ipl)321 x2apic_send_ipi(int cpun, int ipl)
322 {
323 int vector;
324 ulong_t flag;
325
326 ASSERT(apic_mode == LOCAL_X2APIC);
327
328 /*
329 * With X2APIC, Intel relaxed the semantics of the
330 * WRMSR instruction such that references to the X2APIC
331 * MSR registers are no longer serializing instructions.
332 * The code that initiates IPIs assumes that some sort
333 * of memory serialization occurs. The old APIC code
334 * did a write to uncachable memory mapped registers.
335 * Any reference to uncached memory is a serializing
336 * operation. To mimic those semantics here, we do an
337 * atomic operation, which translates to a LOCK OR instruction,
338 * which is serializing.
339 */
340 atomic_or_ulong(&flag, 1);
341
342 vector = apic_resv_vector[ipl];
343
344 flag = intr_clear();
345
346 /*
347 * According to X2APIC specification in section '2.3.5.1' of
348 * Interrupt Command Register Semantics, the semantics of
349 * programming Interrupt Command Register to dispatch an interrupt
350 * is simplified. A single MSR write to the 64-bit ICR is required
351 * for dispatching an interrupt. Specifically with the 64-bit MSR
352 * interface to ICR, system software is not required to check the
353 * status of the delivery status bit prior to writing to the ICR
354 * to send an IPI. With the removal of the Delivery Status bit,
355 * system software no longer has a reason to read the ICR. It remains
356 * readable only to aid in debugging.
357 */
358 #ifdef DEBUG
359 APIC_AV_PENDING_SET();
360 #endif /* DEBUG */
361
362 if ((cpun == psm_get_cpu_id())) {
363 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
364 } else {
365 apic_reg_ops->apic_write_int_cmd(
366 apic_cpus[cpun].aci_local_id, vector);
367 }
368
369 intr_restore(flag);
370 }
371
372 /*
373 * Generates IPI to another CPU depending on the local APIC mode.
374 * apic_send_ipi() and x2apic_send_ipi() depends on the configured
375 * mode of the local APIC, but that may not match the actual mode
376 * early in CPU startup.
377 *
378 * Any changes made to this routine must be accompanied by similar
379 * changes to apic_send_ipi().
380 */
381 void
apic_common_send_ipi(int cpun,int ipl)382 apic_common_send_ipi(int cpun, int ipl)
383 {
384 int vector;
385 ulong_t flag;
386 int mode = apic_local_mode();
387
388 if (mode == LOCAL_X2APIC) {
389 x2apic_send_ipi(cpun, ipl);
390 return;
391 }
392
393 ASSERT(mode == LOCAL_APIC);
394
395 vector = apic_resv_vector[ipl];
396 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
397 flag = intr_clear();
398 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
399 apic_ret();
400 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
401 vector);
402 intr_restore(flag);
403 }
404