1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/md_var.h> 45 #include <machine/smp.h> 46 47 #include <x86/apicreg.h> 48 #include <x86/apicvar.h> 49 50 #include <xen/xen-os.h> 51 #include <xen/features.h> 52 #include <xen/gnttab.h> 53 #include <xen/hypervisor.h> 54 #include <xen/hvm.h> 55 #include <xen/xen_intr.h> 56 57 #include <xen/interface/vcpu.h> 58 59 /*--------------------------------- Macros -----------------------------------*/ 60 61 #define XEN_APIC_UNSUPPORTED \ 62 panic("%s: not available in Xen PV port.", __func__) 63 64 65 /*--------------------------- Forward Declarations ---------------------------*/ 66 #ifdef SMP 67 static driver_filter_t xen_smp_rendezvous_action; 68 #ifdef __amd64__ 69 static driver_filter_t xen_invlop; 70 #else 71 static driver_filter_t xen_invltlb; 72 static driver_filter_t xen_invlpg; 73 static driver_filter_t xen_invlrng; 74 static driver_filter_t xen_invlcache; 75 #endif 76 static driver_filter_t xen_ipi_bitmap_handler; 77 static driver_filter_t xen_cpustop_handler; 78 static driver_filter_t xen_cpususpend_handler; 79 static driver_filter_t xen_ipi_swi_handler; 80 #endif 81 82 /*---------------------------------- Macros ----------------------------------*/ 83 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 84 85 /*--------------------------------- Xen IPIs ---------------------------------*/ 86 #ifdef SMP 87 struct xen_ipi_handler 88 { 89 driver_filter_t *filter; 90 const char *description; 91 }; 92 93 static struct xen_ipi_handler xen_ipis[] = 94 { 95 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 96 #ifdef __amd64__ 97 [IPI_TO_IDX(IPI_INVLOP)] = { xen_invlop, "itlb"}, 98 #else 99 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 100 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 101 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 102 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 103 #endif 104 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 105 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 106 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 107 [IPI_TO_IDX(IPI_SWI)] = { xen_ipi_swi_handler, "sw" }, 108 }; 109 #endif 110 111 /*------------------------------- Per-CPU Data -------------------------------*/ 112 #ifdef SMP 113 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 114 #endif 115 116 /*------------------------------- Xen PV APIC --------------------------------*/ 117 118 static void 119 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 120 { 121 #ifdef SMP 122 cpu_add(apic_id, boot_cpu); 123 #endif 124 } 125 126 static void 127 xen_pv_lapic_init(vm_paddr_t addr) 128 { 129 130 } 131 132 static void 133 xen_pv_lapic_setup(int boot) 134 { 135 136 } 137 138 static void 139 xen_pv_lapic_dump(const char *str) 140 { 141 142 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 143 } 144 145 static void 146 xen_pv_lapic_disable(void) 147 { 148 149 } 150 151 static bool 152 xen_pv_lapic_is_x2apic(void) 153 { 154 155 return (false); 156 } 157 158 static void 159 xen_pv_lapic_eoi(void) 160 { 161 162 XEN_APIC_UNSUPPORTED; 163 } 164 165 static int 166 xen_pv_lapic_id(void) 167 { 168 169 return (PCPU_GET(apic_id)); 170 } 171 172 static int 173 xen_pv_lapic_intr_pending(u_int vector) 174 { 175 176 XEN_APIC_UNSUPPORTED; 177 return (0); 178 } 179 180 static u_int 181 xen_pv_apic_cpuid(u_int apic_id) 182 { 183 #ifdef SMP 184 return (apic_cpuids[apic_id]); 185 #else 186 return (0); 187 #endif 188 } 189 190 static u_int 191 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 192 { 193 194 XEN_APIC_UNSUPPORTED; 195 return (0); 196 } 197 198 static u_int 199 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 200 { 201 202 XEN_APIC_UNSUPPORTED; 203 return (0); 204 } 205 206 static void 207 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 208 { 209 210 XEN_APIC_UNSUPPORTED; 211 } 212 213 static void 214 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 215 { 216 217 XEN_APIC_UNSUPPORTED; 218 } 219 220 static void 221 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 222 { 223 224 XEN_APIC_UNSUPPORTED; 225 } 226 227 static void 228 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 229 { 230 231 XEN_APIC_UNSUPPORTED; 232 } 233 234 static int 235 xen_pv_lapic_enable_pmc(void) 236 { 237 238 XEN_APIC_UNSUPPORTED; 239 return (0); 240 } 241 242 static void 243 xen_pv_lapic_disable_pmc(void) 244 { 245 246 XEN_APIC_UNSUPPORTED; 247 } 248 249 static void 250 xen_pv_lapic_reenable_pmc(void) 251 { 252 253 XEN_APIC_UNSUPPORTED; 254 } 255 256 static void 257 xen_pv_lapic_enable_cmc(void) 258 { 259 260 } 261 262 #ifdef SMP 263 static void 264 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 265 { 266 267 XEN_APIC_UNSUPPORTED; 268 } 269 270 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) 271 static void 272 send_nmi(int dest) 273 { 274 unsigned int cpu; 275 276 /* 277 * NMIs are not routed over event channels, and instead delivered as on 278 * native using the exception vector (#2). Triggering them can be done 279 * using the local APIC, or an hypercall as a shortcut like it's done 280 * below. 281 */ 282 switch(dest) { 283 case APIC_IPI_DEST_SELF: 284 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); 285 break; 286 case APIC_IPI_DEST_ALL: 287 CPU_FOREACH(cpu) 288 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 289 PCPU_ID_GET(cpu, vcpu_id), NULL); 290 break; 291 case APIC_IPI_DEST_OTHERS: 292 CPU_FOREACH(cpu) 293 if (cpu != PCPU_GET(cpuid)) 294 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 295 PCPU_ID_GET(cpu, vcpu_id), NULL); 296 break; 297 default: 298 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 299 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); 300 break; 301 } 302 } 303 #undef PCPU_ID_GET 304 305 static void 306 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 307 { 308 xen_intr_handle_t *ipi_handle; 309 int ipi_idx, to_cpu, self; 310 311 if (vector >= IPI_NMI_FIRST) { 312 send_nmi(dest); 313 return; 314 } 315 316 ipi_idx = IPI_TO_IDX(vector); 317 if (ipi_idx >= nitems(xen_ipis)) 318 panic("IPI out of range"); 319 320 switch(dest) { 321 case APIC_IPI_DEST_SELF: 322 ipi_handle = DPCPU_GET(ipi_handle); 323 xen_intr_signal(ipi_handle[ipi_idx]); 324 break; 325 case APIC_IPI_DEST_ALL: 326 CPU_FOREACH(to_cpu) { 327 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 328 xen_intr_signal(ipi_handle[ipi_idx]); 329 } 330 break; 331 case APIC_IPI_DEST_OTHERS: 332 self = PCPU_GET(cpuid); 333 CPU_FOREACH(to_cpu) { 334 if (to_cpu != self) { 335 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 336 xen_intr_signal(ipi_handle[ipi_idx]); 337 } 338 } 339 break; 340 default: 341 to_cpu = apic_cpuid(dest); 342 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 343 xen_intr_signal(ipi_handle[ipi_idx]); 344 break; 345 } 346 } 347 348 static int 349 xen_pv_lapic_ipi_wait(int delay) 350 { 351 352 XEN_APIC_UNSUPPORTED; 353 return (0); 354 } 355 #endif /* SMP */ 356 357 static int 358 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 359 { 360 361 XEN_APIC_UNSUPPORTED; 362 return (-1); 363 } 364 365 static void 366 xen_pv_lapic_ipi_free(int vector) 367 { 368 369 XEN_APIC_UNSUPPORTED; 370 } 371 372 static int 373 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 374 { 375 376 XEN_APIC_UNSUPPORTED; 377 return (0); 378 } 379 380 static int 381 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 382 { 383 384 XEN_APIC_UNSUPPORTED; 385 return (0); 386 } 387 388 static int 389 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 390 { 391 392 XEN_APIC_UNSUPPORTED; 393 return (0); 394 } 395 396 static int 397 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 398 enum intr_trigger trigger) 399 { 400 401 XEN_APIC_UNSUPPORTED; 402 return (0); 403 } 404 405 /* Xen apic_ops implementation */ 406 struct apic_ops xen_apic_ops = { 407 .create = xen_pv_lapic_create, 408 .init = xen_pv_lapic_init, 409 .xapic_mode = xen_pv_lapic_disable, 410 .is_x2apic = xen_pv_lapic_is_x2apic, 411 .setup = xen_pv_lapic_setup, 412 .dump = xen_pv_lapic_dump, 413 .disable = xen_pv_lapic_disable, 414 .eoi = xen_pv_lapic_eoi, 415 .id = xen_pv_lapic_id, 416 .intr_pending = xen_pv_lapic_intr_pending, 417 .set_logical_id = xen_pv_lapic_set_logical_id, 418 .cpuid = xen_pv_apic_cpuid, 419 .alloc_vector = xen_pv_apic_alloc_vector, 420 .alloc_vectors = xen_pv_apic_alloc_vectors, 421 .enable_vector = xen_pv_apic_enable_vector, 422 .disable_vector = xen_pv_apic_disable_vector, 423 .free_vector = xen_pv_apic_free_vector, 424 .enable_pmc = xen_pv_lapic_enable_pmc, 425 .disable_pmc = xen_pv_lapic_disable_pmc, 426 .reenable_pmc = xen_pv_lapic_reenable_pmc, 427 .enable_cmc = xen_pv_lapic_enable_cmc, 428 #ifdef SMP 429 .ipi_raw = xen_pv_lapic_ipi_raw, 430 .ipi_vectored = xen_pv_lapic_ipi_vectored, 431 .ipi_wait = xen_pv_lapic_ipi_wait, 432 #endif 433 .ipi_alloc = xen_pv_lapic_ipi_alloc, 434 .ipi_free = xen_pv_lapic_ipi_free, 435 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 436 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 437 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 438 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 439 }; 440 441 #ifdef SMP 442 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 443 /* 444 * These are C clones of the ASM functions found in apic_vector. 445 */ 446 static int 447 xen_ipi_bitmap_handler(void *arg) 448 { 449 struct trapframe *frame; 450 451 frame = arg; 452 ipi_bitmap_handler(*frame); 453 return (FILTER_HANDLED); 454 } 455 456 static int 457 xen_smp_rendezvous_action(void *arg) 458 { 459 #ifdef COUNT_IPIS 460 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 461 #endif /* COUNT_IPIS */ 462 463 smp_rendezvous_action(); 464 return (FILTER_HANDLED); 465 } 466 467 #ifdef __amd64__ 468 static int 469 xen_invlop(void *arg) 470 { 471 472 invlop_handler(); 473 return (FILTER_HANDLED); 474 } 475 476 #else /* __i386__ */ 477 478 static int 479 xen_invltlb(void *arg) 480 { 481 482 invltlb_handler(); 483 return (FILTER_HANDLED); 484 } 485 486 static int 487 xen_invlpg(void *arg) 488 { 489 490 invlpg_handler(); 491 return (FILTER_HANDLED); 492 } 493 494 static int 495 xen_invlrng(void *arg) 496 { 497 498 invlrng_handler(); 499 return (FILTER_HANDLED); 500 } 501 502 static int 503 xen_invlcache(void *arg) 504 { 505 506 invlcache_handler(); 507 return (FILTER_HANDLED); 508 } 509 #endif /* __amd64__ */ 510 511 static int 512 xen_cpustop_handler(void *arg) 513 { 514 515 cpustop_handler(); 516 return (FILTER_HANDLED); 517 } 518 519 static int 520 xen_cpususpend_handler(void *arg) 521 { 522 523 cpususpend_handler(); 524 return (FILTER_HANDLED); 525 } 526 527 static int 528 xen_ipi_swi_handler(void *arg) 529 { 530 struct trapframe *frame = arg; 531 532 ipi_swi_handler(*frame); 533 return (FILTER_HANDLED); 534 } 535 536 /*----------------------------- XEN PV IPI setup -----------------------------*/ 537 /* 538 * Those functions are provided outside of the Xen PV APIC implementation 539 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 540 * because on PVHVM there's an emulated LAPIC provided by Xen. 541 */ 542 static void 543 xen_cpu_ipi_init(int cpu) 544 { 545 xen_intr_handle_t *ipi_handle; 546 const struct xen_ipi_handler *ipi; 547 int idx, rc; 548 549 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 550 551 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 552 553 if (ipi->filter == NULL) { 554 ipi_handle[idx] = NULL; 555 continue; 556 } 557 558 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 559 INTR_TYPE_TTY, &ipi_handle[idx]); 560 if (rc != 0) 561 panic("Unable to allocate a XEN IPI port"); 562 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 563 } 564 } 565 566 static void 567 xen_setup_cpus(void) 568 { 569 int i; 570 571 if (!xen_vector_callback_enabled) 572 return; 573 574 CPU_FOREACH(i) 575 xen_cpu_ipi_init(i); 576 577 /* Set the xen pv ipi ops to replace the native ones */ 578 if (xen_hvm_domain()) 579 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 580 } 581 582 /* Switch to using PV IPIs as soon as the vcpu_id is set. */ 583 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL); 584 #endif /* SMP */ 585