1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/md_var.h> 45 #include <machine/smp.h> 46 47 #include <x86/apicreg.h> 48 #include <x86/apicvar.h> 49 50 #include <xen/xen-os.h> 51 #include <xen/features.h> 52 #include <xen/gnttab.h> 53 #include <xen/hypervisor.h> 54 #include <xen/hvm.h> 55 #include <xen/xen_intr.h> 56 57 #include <contrib/xen/arch-x86/cpuid.h> 58 #include <contrib/xen/vcpu.h> 59 60 /*--------------------------- Forward Declarations ---------------------------*/ 61 static driver_filter_t xen_smp_rendezvous_action; 62 #ifdef __amd64__ 63 static driver_filter_t xen_invlop; 64 #else 65 static driver_filter_t xen_invltlb; 66 static driver_filter_t xen_invlpg; 67 static driver_filter_t xen_invlrng; 68 static driver_filter_t xen_invlcache; 69 #endif 70 static driver_filter_t xen_ipi_bitmap_handler; 71 static driver_filter_t xen_cpustop_handler; 72 static driver_filter_t xen_cpususpend_handler; 73 static driver_filter_t xen_ipi_swi_handler; 74 75 /*---------------------------------- Macros ----------------------------------*/ 76 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 77 78 /*--------------------------------- Xen IPIs ---------------------------------*/ 79 struct xen_ipi_handler 80 { 81 driver_filter_t *filter; 82 const char *description; 83 }; 84 85 static struct xen_ipi_handler xen_ipis[] = 86 { 87 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 88 #ifdef __amd64__ 89 [IPI_TO_IDX(IPI_INVLOP)] = { xen_invlop, "itlb"}, 90 #else 91 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 92 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 93 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 94 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 95 #endif 96 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 97 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 98 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 99 [IPI_TO_IDX(IPI_SWI)] = { xen_ipi_swi_handler, "sw" }, 100 }; 101 102 /* 103 * Save previous (native) handler as a fallback. Xen < 4.7 doesn't support 104 * VCPUOP_send_nmi for HVM guests, and thus we need a fallback in that case: 105 * 106 * https://lists.freebsd.org/archives/freebsd-xen/2022-January/000032.html 107 */ 108 void (*native_ipi_vectored)(u_int, int); 109 110 /*------------------------------- Per-CPU Data -------------------------------*/ 111 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 112 113 /*------------------------------- Xen PV APIC --------------------------------*/ 114 115 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) 116 static int 117 send_nmi(int dest) 118 { 119 unsigned int cpu; 120 int rc = 0; 121 122 /* 123 * NMIs are not routed over event channels, and instead delivered as on 124 * native using the exception vector (#2). Triggering them can be done 125 * using the local APIC, or an hypercall as a shortcut like it's done 126 * below. 127 */ 128 switch(dest) { 129 case APIC_IPI_DEST_SELF: 130 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); 131 break; 132 case APIC_IPI_DEST_ALL: 133 CPU_FOREACH(cpu) { 134 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 135 PCPU_ID_GET(cpu, vcpu_id), NULL); 136 if (rc != 0) 137 break; 138 } 139 break; 140 case APIC_IPI_DEST_OTHERS: 141 CPU_FOREACH(cpu) { 142 if (cpu != PCPU_GET(cpuid)) { 143 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 144 PCPU_ID_GET(cpu, vcpu_id), NULL); 145 if (rc != 0) 146 break; 147 } 148 } 149 break; 150 default: 151 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 152 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); 153 break; 154 } 155 156 return rc; 157 } 158 #undef PCPU_ID_GET 159 160 static void 161 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 162 { 163 xen_intr_handle_t *ipi_handle; 164 int ipi_idx, to_cpu, self; 165 static bool pvnmi = true; 166 167 if (vector >= IPI_NMI_FIRST) { 168 if (pvnmi) { 169 int rc = send_nmi(dest); 170 171 if (rc != 0) { 172 printf( 173 "Sending NMI using hypercall failed (%d) switching to APIC\n", rc); 174 pvnmi = false; 175 native_ipi_vectored(vector, dest); 176 } 177 } else 178 native_ipi_vectored(vector, dest); 179 180 return; 181 } 182 183 ipi_idx = IPI_TO_IDX(vector); 184 if (ipi_idx >= nitems(xen_ipis)) 185 panic("IPI out of range"); 186 187 switch(dest) { 188 case APIC_IPI_DEST_SELF: 189 ipi_handle = DPCPU_GET(ipi_handle); 190 xen_intr_signal(ipi_handle[ipi_idx]); 191 break; 192 case APIC_IPI_DEST_ALL: 193 CPU_FOREACH(to_cpu) { 194 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 195 xen_intr_signal(ipi_handle[ipi_idx]); 196 } 197 break; 198 case APIC_IPI_DEST_OTHERS: 199 self = PCPU_GET(cpuid); 200 CPU_FOREACH(to_cpu) { 201 if (to_cpu != self) { 202 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 203 xen_intr_signal(ipi_handle[ipi_idx]); 204 } 205 } 206 break; 207 default: 208 to_cpu = apic_cpuid(dest); 209 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 210 xen_intr_signal(ipi_handle[ipi_idx]); 211 break; 212 } 213 } 214 215 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 216 /* 217 * These are C clones of the ASM functions found in apic_vector. 218 */ 219 static int 220 xen_ipi_bitmap_handler(void *arg) 221 { 222 struct trapframe *frame; 223 224 frame = arg; 225 ipi_bitmap_handler(*frame); 226 return (FILTER_HANDLED); 227 } 228 229 static int 230 xen_smp_rendezvous_action(void *arg) 231 { 232 #ifdef COUNT_IPIS 233 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 234 #endif /* COUNT_IPIS */ 235 236 smp_rendezvous_action(); 237 return (FILTER_HANDLED); 238 } 239 240 #ifdef __amd64__ 241 static int 242 xen_invlop(void *arg) 243 { 244 245 invlop_handler(); 246 return (FILTER_HANDLED); 247 } 248 249 #else /* __i386__ */ 250 251 static int 252 xen_invltlb(void *arg) 253 { 254 255 invltlb_handler(); 256 return (FILTER_HANDLED); 257 } 258 259 static int 260 xen_invlpg(void *arg) 261 { 262 263 invlpg_handler(); 264 return (FILTER_HANDLED); 265 } 266 267 static int 268 xen_invlrng(void *arg) 269 { 270 271 invlrng_handler(); 272 return (FILTER_HANDLED); 273 } 274 275 static int 276 xen_invlcache(void *arg) 277 { 278 279 invlcache_handler(); 280 return (FILTER_HANDLED); 281 } 282 #endif /* __amd64__ */ 283 284 static int 285 xen_cpustop_handler(void *arg) 286 { 287 288 cpustop_handler(); 289 return (FILTER_HANDLED); 290 } 291 292 static int 293 xen_cpususpend_handler(void *arg) 294 { 295 296 cpususpend_handler(); 297 return (FILTER_HANDLED); 298 } 299 300 static int 301 xen_ipi_swi_handler(void *arg) 302 { 303 struct trapframe *frame = arg; 304 305 ipi_swi_handler(*frame); 306 return (FILTER_HANDLED); 307 } 308 309 /*----------------------------- XEN PV IPI setup -----------------------------*/ 310 /* 311 * Those functions are provided outside of the Xen PV APIC implementation 312 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 313 * because on PVHVM there's an emulated LAPIC provided by Xen. 314 */ 315 static void 316 xen_cpu_ipi_init(int cpu) 317 { 318 xen_intr_handle_t *ipi_handle; 319 const struct xen_ipi_handler *ipi; 320 int idx, rc; 321 322 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 323 324 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 325 if (ipi->filter == NULL) { 326 ipi_handle[idx] = NULL; 327 continue; 328 } 329 330 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 331 INTR_TYPE_TTY, &ipi_handle[idx]); 332 if (rc != 0) 333 panic("Unable to allocate a XEN IPI port"); 334 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 335 } 336 } 337 338 static void 339 xen_setup_cpus(void) 340 { 341 uint32_t regs[4]; 342 int i; 343 344 if (!xen_vector_callback_enabled) 345 return; 346 347 /* 348 * Check whether the APIC virtualization is hardware assisted, as 349 * that's faster than using event channels because it avoids the VM 350 * exit. 351 */ 352 KASSERT(xen_cpuid_base != 0, ("Invalid base Xen CPUID leaf")); 353 cpuid_count(xen_cpuid_base + 4, 0, regs); 354 if ((x2apic_mode && (regs[0] & XEN_HVM_CPUID_X2APIC_VIRT)) || 355 (!x2apic_mode && (regs[0] & XEN_HVM_CPUID_APIC_ACCESS_VIRT))) 356 return; 357 358 CPU_FOREACH(i) 359 xen_cpu_ipi_init(i); 360 361 /* Set the xen pv ipi ops to replace the native ones */ 362 ipi_vectored = xen_pv_lapic_ipi_vectored; 363 native_ipi_vectored = ipi_vectored; 364 } 365 366 /* Switch to using PV IPIs as soon as the vcpu_id is set. */ 367 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL); 368