1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25 /*
26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 * Copyright (c) 2014 by Delphix. All rights reserved.
28 * Copyright 2018 Joyent, Inc.
29 */
30
31 #include <sys/cpuvar.h>
32 #include <sys/psm.h>
33 #include <sys/archsystm.h>
34 #include <sys/apic.h>
35 #include <sys/apic_common.h>
36 #include <sys/sunddi.h>
37 #include <sys/ddi_impldefs.h>
38 #include <sys/mach_intr.h>
39 #include <sys/sysmacros.h>
40 #include <sys/trap.h>
41 #include <sys/x86_archext.h>
42 #include <sys/privregs.h>
43 #include <sys/psm_common.h>
44
45 /* Function prototypes of X2APIC */
46 static uint64_t local_x2apic_read(uint32_t msr);
47 static void local_x2apic_write(uint32_t msr, uint64_t value);
48 static int get_local_x2apic_pri(void);
49 static void local_x2apic_write_task_reg(uint64_t value);
50 static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
51
52 /*
53 * According to the X2APIC specification:
54 *
55 * xAPIC global enable X2APIC enable Description
56 * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
57 * -----------------------------------------------------------
58 * 0 0 APIC is disabled
59 * 0 1 Invalid
60 * 1 0 APIC is enabled in xAPIC mode
61 * 1 1 APIC is enabled in X2APIC mode
62 * -----------------------------------------------------------
63 */
64 int x2apic_enable = 1;
65
66 /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
67 static apic_reg_ops_t x2apic_regs_ops = {
68 local_x2apic_read,
69 local_x2apic_write,
70 get_local_x2apic_pri,
71 local_x2apic_write_task_reg,
72 local_x2apic_write_int_cmd,
73 apic_send_EOI,
74 };
75
76 /*
77 * X2APIC Implementation.
78 */
79 static uint64_t
local_x2apic_read(uint32_t msr)80 local_x2apic_read(uint32_t msr)
81 {
82 uint64_t i;
83
84 i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
85 return (i);
86 }
87
88 static void
local_x2apic_write(uint32_t msr,uint64_t value)89 local_x2apic_write(uint32_t msr, uint64_t value)
90 {
91 uint64_t tmp;
92
93 if (msr != APIC_EOI_REG) {
94 tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
95 tmp = (tmp & 0xffffffff00000000) | value;
96 } else {
97 tmp = 0;
98 }
99
100 wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
101 }
102
103 static int
get_local_x2apic_pri(void)104 get_local_x2apic_pri(void)
105 {
106 return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
107 }
108
109 static void
local_x2apic_write_task_reg(uint64_t value)110 local_x2apic_write_task_reg(uint64_t value)
111 {
112 X2APIC_WRITE(APIC_TASK_REG, value);
113 }
114
115 static void
local_x2apic_write_int_cmd(uint32_t cpu_id,uint32_t cmd1)116 local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
117 {
118 wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
119 (((uint64_t)cpu_id << 32) | cmd1));
120 }
121
122 int
apic_detect_x2apic(void)123 apic_detect_x2apic(void)
124 {
125 if (x2apic_enable == 0)
126 return (0);
127
128 return (is_x86_feature(x86_featureset, X86FSET_X2APIC));
129 }
130
131 void
apic_enable_x2apic(void)132 apic_enable_x2apic(void)
133 {
134 uint64_t apic_base_msr;
135
136 if (apic_local_mode() == LOCAL_X2APIC) {
137 /* BIOS apparently has enabled X2APIC */
138 if (apic_mode != LOCAL_X2APIC)
139 x2apic_update_psm();
140 return;
141 }
142
143 /*
144 * This is the first time we are enabling X2APIC on this CPU
145 */
146 apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
147 apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
148 wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
149
150 if (apic_mode != LOCAL_X2APIC)
151 x2apic_update_psm();
152 }
153
154 /*
155 * Change apic_reg_ops depending upon the apic_mode.
156 */
157 void
apic_change_ops()158 apic_change_ops()
159 {
160 if (apic_mode == LOCAL_APIC)
161 apic_reg_ops = &local_apic_regs_ops;
162 else if (apic_mode == LOCAL_X2APIC)
163 apic_reg_ops = &x2apic_regs_ops;
164 }
165
166 /*
167 * Generates an interprocessor interrupt to another CPU when X2APIC mode is
168 * enabled.
169 */
170 void
x2apic_send_ipi(int cpun,int ipl)171 x2apic_send_ipi(int cpun, int ipl)
172 {
173 int vector;
174 ulong_t flag;
175
176 ASSERT(apic_mode == LOCAL_X2APIC);
177
178 /*
179 * With X2APIC, Intel relaxed the semantics of the
180 * WRMSR instruction such that references to the X2APIC
181 * MSR registers are no longer serializing instructions.
182 * The code that initiates IPIs assumes that some sort
183 * of memory serialization occurs. The old APIC code
184 * did a write to uncachable memory mapped registers.
185 * Any reference to uncached memory is a serializing
186 * operation. To mimic those semantics here, we do an
187 * atomic operation, which translates to a LOCK OR instruction,
188 * which is serializing.
189 */
190 atomic_or_ulong(&flag, 1);
191
192 vector = apic_resv_vector[ipl];
193
194 flag = intr_clear();
195
196 /*
197 * According to X2APIC specification in section '2.3.5.1' of
198 * Interrupt Command Register Semantics, the semantics of
199 * programming Interrupt Command Register to dispatch an interrupt
200 * is simplified. A single MSR write to the 64-bit ICR is required
201 * for dispatching an interrupt. Specifically with the 64-bit MSR
202 * interface to ICR, system software is not required to check the
203 * status of the delivery status bit prior to writing to the ICR
204 * to send an IPI. With the removal of the Delivery Status bit,
205 * system software no longer has a reason to read the ICR. It remains
206 * readable only to aid in debugging.
207 */
208 #ifdef DEBUG
209 APIC_AV_PENDING_SET();
210 #endif /* DEBUG */
211
212 if ((cpun == psm_get_cpu_id())) {
213 X2APIC_WRITE(X2APIC_SELF_IPI, vector);
214 } else {
215 apic_reg_ops->apic_write_int_cmd(
216 apic_cpus[cpun].aci_local_id, vector);
217 }
218
219 intr_restore(flag);
220 }
221
222 void
x2apic_send_pir_ipi(processorid_t cpun)223 x2apic_send_pir_ipi(processorid_t cpun)
224 {
225 const int vector = apic_pir_vect;
226 ulong_t flag;
227
228 ASSERT(apic_mode == LOCAL_X2APIC);
229 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
230
231 /* Serialize as described in x2apic_send_ipi() above. */
232 atomic_or_ulong(&flag, 1);
233
234 flag = intr_clear();
235
236 /* Self-IPI for inducing PIR makes no sense. */
237 if ((cpun != psm_get_cpu_id())) {
238 #ifdef DEBUG
239 /* Only for debugging. (again, see: x2apic_send_ipi) */
240 APIC_AV_PENDING_SET();
241 #endif /* DEBUG */
242
243 apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
244 vector);
245 }
246
247 intr_restore(flag);
248 }
249
250 /*
251 * Generates IPI to another CPU depending on the local APIC mode.
252 * apic_send_ipi() and x2apic_send_ipi() depends on the configured
253 * mode of the local APIC, but that may not match the actual mode
254 * early in CPU startup.
255 *
256 * Any changes made to this routine must be accompanied by similar
257 * changes to apic_send_ipi().
258 */
259 void
apic_common_send_ipi(int cpun,int ipl)260 apic_common_send_ipi(int cpun, int ipl)
261 {
262 int vector;
263 ulong_t flag;
264 int mode = apic_local_mode();
265
266 if (mode == LOCAL_X2APIC) {
267 x2apic_send_ipi(cpun, ipl);
268 return;
269 }
270
271 ASSERT(mode == LOCAL_APIC);
272
273 vector = apic_resv_vector[ipl];
274 ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
275 flag = intr_clear();
276 while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
277 apic_ret();
278 local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
279 vector);
280 intr_restore(flag);
281 }
282
283 void
apic_common_send_pir_ipi(processorid_t cpun)284 apic_common_send_pir_ipi(processorid_t cpun)
285 {
286 const int mode = apic_local_mode();
287
288 if (mode == LOCAL_X2APIC) {
289 x2apic_send_pir_ipi(cpun);
290 return;
291 }
292
293 apic_send_pir_ipi(cpun);
294 }
295