1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/smp.h> 45 46 #include <x86/apicreg.h> 47 #include <x86/apicvar.h> 48 49 #include <xen/xen-os.h> 50 #include <xen/features.h> 51 #include <xen/gnttab.h> 52 #include <xen/hypervisor.h> 53 #include <xen/hvm.h> 54 #include <xen/xen_intr.h> 55 56 #include <xen/interface/vcpu.h> 57 58 /*--------------------------------- Macros -----------------------------------*/ 59 60 #define XEN_APIC_UNSUPPORTED \ 61 panic("%s: not available in Xen PV port.", __func__) 62 63 64 /*--------------------------- Forward Declarations ---------------------------*/ 65 #ifdef SMP 66 static driver_filter_t xen_smp_rendezvous_action; 67 static driver_filter_t xen_invltlb; 68 static driver_filter_t xen_invlpg; 69 static driver_filter_t xen_invlrng; 70 static driver_filter_t xen_invlcache; 71 static driver_filter_t xen_ipi_bitmap_handler; 72 static driver_filter_t xen_cpustop_handler; 73 static driver_filter_t xen_cpususpend_handler; 74 static driver_filter_t xen_cpustophard_handler; 75 #endif 76 77 /*---------------------------------- Macros ----------------------------------*/ 78 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 79 80 /*--------------------------------- Xen IPIs ---------------------------------*/ 81 #ifdef SMP 82 struct xen_ipi_handler 83 { 84 driver_filter_t *filter; 85 const char *description; 86 }; 87 88 static struct xen_ipi_handler xen_ipis[] = 89 { 90 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 91 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 92 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 93 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 94 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 95 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 96 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 97 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 98 [IPI_TO_IDX(IPI_STOP_HARD)] = { xen_cpustophard_handler, "sth" }, 99 }; 100 #endif 101 102 /*------------------------------- Per-CPU Data -------------------------------*/ 103 #ifdef SMP 104 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 105 #endif 106 107 /*------------------------------- Xen PV APIC --------------------------------*/ 108 109 static void 110 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 111 { 112 #ifdef SMP 113 cpu_add(apic_id, boot_cpu); 114 #endif 115 } 116 117 static void 118 xen_pv_lapic_init(vm_paddr_t addr) 119 { 120 121 } 122 123 static void 124 xen_pv_lapic_setup(int boot) 125 { 126 127 } 128 129 static void 130 xen_pv_lapic_dump(const char *str) 131 { 132 133 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 134 } 135 136 static void 137 xen_pv_lapic_disable(void) 138 { 139 140 } 141 142 static bool 143 xen_pv_lapic_is_x2apic(void) 144 { 145 146 return (false); 147 } 148 149 static void 150 xen_pv_lapic_eoi(void) 151 { 152 153 XEN_APIC_UNSUPPORTED; 154 } 155 156 static int 157 xen_pv_lapic_id(void) 158 { 159 160 return (PCPU_GET(apic_id)); 161 } 162 163 static int 164 xen_pv_lapic_intr_pending(u_int vector) 165 { 166 167 XEN_APIC_UNSUPPORTED; 168 return (0); 169 } 170 171 static u_int 172 xen_pv_apic_cpuid(u_int apic_id) 173 { 174 #ifdef SMP 175 return (apic_cpuids[apic_id]); 176 #else 177 return (0); 178 #endif 179 } 180 181 static u_int 182 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 183 { 184 185 XEN_APIC_UNSUPPORTED; 186 return (0); 187 } 188 189 static u_int 190 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 191 { 192 193 XEN_APIC_UNSUPPORTED; 194 return (0); 195 } 196 197 static void 198 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 199 { 200 201 XEN_APIC_UNSUPPORTED; 202 } 203 204 static void 205 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 206 { 207 208 XEN_APIC_UNSUPPORTED; 209 } 210 211 static void 212 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 213 { 214 215 XEN_APIC_UNSUPPORTED; 216 } 217 218 static void 219 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 220 { 221 222 XEN_APIC_UNSUPPORTED; 223 } 224 225 static int 226 xen_pv_lapic_enable_pmc(void) 227 { 228 229 XEN_APIC_UNSUPPORTED; 230 return (0); 231 } 232 233 static void 234 xen_pv_lapic_disable_pmc(void) 235 { 236 237 XEN_APIC_UNSUPPORTED; 238 } 239 240 static void 241 xen_pv_lapic_reenable_pmc(void) 242 { 243 244 XEN_APIC_UNSUPPORTED; 245 } 246 247 static void 248 xen_pv_lapic_enable_cmc(void) 249 { 250 251 } 252 253 #ifdef SMP 254 static void 255 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 256 { 257 258 XEN_APIC_UNSUPPORTED; 259 } 260 261 static void 262 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 263 { 264 xen_intr_handle_t *ipi_handle; 265 int ipi_idx, to_cpu, self; 266 267 ipi_idx = IPI_TO_IDX(vector); 268 if (ipi_idx >= nitems(xen_ipis)) 269 panic("IPI out of range"); 270 271 switch(dest) { 272 case APIC_IPI_DEST_SELF: 273 ipi_handle = DPCPU_GET(ipi_handle); 274 xen_intr_signal(ipi_handle[ipi_idx]); 275 break; 276 case APIC_IPI_DEST_ALL: 277 CPU_FOREACH(to_cpu) { 278 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 279 xen_intr_signal(ipi_handle[ipi_idx]); 280 } 281 break; 282 case APIC_IPI_DEST_OTHERS: 283 self = PCPU_GET(cpuid); 284 CPU_FOREACH(to_cpu) { 285 if (to_cpu != self) { 286 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 287 xen_intr_signal(ipi_handle[ipi_idx]); 288 } 289 } 290 break; 291 default: 292 to_cpu = apic_cpuid(dest); 293 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 294 xen_intr_signal(ipi_handle[ipi_idx]); 295 break; 296 } 297 } 298 299 static int 300 xen_pv_lapic_ipi_wait(int delay) 301 { 302 303 XEN_APIC_UNSUPPORTED; 304 return (0); 305 } 306 #endif /* SMP */ 307 308 static int 309 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 310 { 311 312 XEN_APIC_UNSUPPORTED; 313 return (-1); 314 } 315 316 static void 317 xen_pv_lapic_ipi_free(int vector) 318 { 319 320 XEN_APIC_UNSUPPORTED; 321 } 322 323 static int 324 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 325 { 326 327 XEN_APIC_UNSUPPORTED; 328 return (0); 329 } 330 331 static int 332 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 333 { 334 335 XEN_APIC_UNSUPPORTED; 336 return (0); 337 } 338 339 static int 340 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 341 { 342 343 XEN_APIC_UNSUPPORTED; 344 return (0); 345 } 346 347 static int 348 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 349 enum intr_trigger trigger) 350 { 351 352 XEN_APIC_UNSUPPORTED; 353 return (0); 354 } 355 356 /* Xen apic_ops implementation */ 357 struct apic_ops xen_apic_ops = { 358 .create = xen_pv_lapic_create, 359 .init = xen_pv_lapic_init, 360 .xapic_mode = xen_pv_lapic_disable, 361 .is_x2apic = xen_pv_lapic_is_x2apic, 362 .setup = xen_pv_lapic_setup, 363 .dump = xen_pv_lapic_dump, 364 .disable = xen_pv_lapic_disable, 365 .eoi = xen_pv_lapic_eoi, 366 .id = xen_pv_lapic_id, 367 .intr_pending = xen_pv_lapic_intr_pending, 368 .set_logical_id = xen_pv_lapic_set_logical_id, 369 .cpuid = xen_pv_apic_cpuid, 370 .alloc_vector = xen_pv_apic_alloc_vector, 371 .alloc_vectors = xen_pv_apic_alloc_vectors, 372 .enable_vector = xen_pv_apic_enable_vector, 373 .disable_vector = xen_pv_apic_disable_vector, 374 .free_vector = xen_pv_apic_free_vector, 375 .enable_pmc = xen_pv_lapic_enable_pmc, 376 .disable_pmc = xen_pv_lapic_disable_pmc, 377 .reenable_pmc = xen_pv_lapic_reenable_pmc, 378 .enable_cmc = xen_pv_lapic_enable_cmc, 379 #ifdef SMP 380 .ipi_raw = xen_pv_lapic_ipi_raw, 381 .ipi_vectored = xen_pv_lapic_ipi_vectored, 382 .ipi_wait = xen_pv_lapic_ipi_wait, 383 #endif 384 .ipi_alloc = xen_pv_lapic_ipi_alloc, 385 .ipi_free = xen_pv_lapic_ipi_free, 386 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 387 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 388 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 389 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 390 }; 391 392 #ifdef SMP 393 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 394 /* 395 * These are C clones of the ASM functions found in apic_vector. 396 */ 397 static int 398 xen_ipi_bitmap_handler(void *arg) 399 { 400 struct trapframe *frame; 401 402 frame = arg; 403 ipi_bitmap_handler(*frame); 404 return (FILTER_HANDLED); 405 } 406 407 static int 408 xen_smp_rendezvous_action(void *arg) 409 { 410 #ifdef COUNT_IPIS 411 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 412 #endif /* COUNT_IPIS */ 413 414 smp_rendezvous_action(); 415 return (FILTER_HANDLED); 416 } 417 418 static int 419 xen_invltlb(void *arg) 420 { 421 422 invltlb_handler(); 423 return (FILTER_HANDLED); 424 } 425 426 #ifdef __amd64__ 427 static int 428 xen_invltlb_invpcid(void *arg) 429 { 430 431 invltlb_invpcid_handler(); 432 return (FILTER_HANDLED); 433 } 434 435 static int 436 xen_invltlb_pcid(void *arg) 437 { 438 439 invltlb_pcid_handler(); 440 return (FILTER_HANDLED); 441 } 442 #endif 443 444 static int 445 xen_invlpg(void *arg) 446 { 447 448 invlpg_handler(); 449 return (FILTER_HANDLED); 450 } 451 452 static int 453 xen_invlrng(void *arg) 454 { 455 456 invlrng_handler(); 457 return (FILTER_HANDLED); 458 } 459 460 static int 461 xen_invlcache(void *arg) 462 { 463 464 invlcache_handler(); 465 return (FILTER_HANDLED); 466 } 467 468 static int 469 xen_cpustop_handler(void *arg) 470 { 471 472 cpustop_handler(); 473 return (FILTER_HANDLED); 474 } 475 476 static int 477 xen_cpususpend_handler(void *arg) 478 { 479 480 cpususpend_handler(); 481 return (FILTER_HANDLED); 482 } 483 484 static int 485 xen_cpustophard_handler(void *arg) 486 { 487 488 ipi_nmi_handler(); 489 return (FILTER_HANDLED); 490 } 491 492 /*----------------------------- XEN PV IPI setup -----------------------------*/ 493 /* 494 * Those functions are provided outside of the Xen PV APIC implementation 495 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 496 * because on PVHVM there's an emulated LAPIC provided by Xen. 497 */ 498 static void 499 xen_cpu_ipi_init(int cpu) 500 { 501 xen_intr_handle_t *ipi_handle; 502 const struct xen_ipi_handler *ipi; 503 int idx, rc; 504 505 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 506 507 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 508 509 if (ipi->filter == NULL) { 510 ipi_handle[idx] = NULL; 511 continue; 512 } 513 514 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 515 INTR_TYPE_TTY, &ipi_handle[idx]); 516 if (rc != 0) 517 panic("Unable to allocate a XEN IPI port"); 518 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 519 } 520 } 521 522 static void 523 xen_setup_cpus(void) 524 { 525 int i; 526 527 if (!xen_vector_callback_enabled) 528 return; 529 530 #ifdef __amd64__ 531 if (pmap_pcid_enabled) { 532 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = invpcid_works ? 533 xen_invltlb_invpcid : xen_invltlb_pcid; 534 } 535 #endif 536 CPU_FOREACH(i) 537 xen_cpu_ipi_init(i); 538 539 /* Set the xen pv ipi ops to replace the native ones */ 540 if (xen_hvm_domain()) 541 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 542 } 543 544 /* We need to setup IPIs before APs are started */ 545 SYSINIT(xen_setup_cpus, SI_SUB_SMP-1, SI_ORDER_FIRST, xen_setup_cpus, NULL); 546 #endif /* SMP */ 547