1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/md_var.h> 45 #include <machine/smp.h> 46 47 #include <x86/apicreg.h> 48 #include <x86/apicvar.h> 49 50 #include <xen/xen-os.h> 51 #include <xen/features.h> 52 #include <xen/gnttab.h> 53 #include <xen/hypervisor.h> 54 #include <xen/hvm.h> 55 #include <xen/xen_intr.h> 56 57 #include <xen/interface/vcpu.h> 58 59 /*--------------------------------- Macros -----------------------------------*/ 60 61 #define XEN_APIC_UNSUPPORTED \ 62 panic("%s: not available in Xen PV port.", __func__) 63 64 /*--------------------------- Forward Declarations ---------------------------*/ 65 #ifdef SMP 66 static driver_filter_t xen_smp_rendezvous_action; 67 #ifdef __amd64__ 68 static driver_filter_t xen_invlop; 69 #else 70 static driver_filter_t xen_invltlb; 71 static driver_filter_t xen_invlpg; 72 static driver_filter_t xen_invlrng; 73 static driver_filter_t xen_invlcache; 74 #endif 75 static driver_filter_t xen_ipi_bitmap_handler; 76 static driver_filter_t xen_cpustop_handler; 77 static driver_filter_t xen_cpususpend_handler; 78 static driver_filter_t xen_ipi_swi_handler; 79 #endif 80 81 /*---------------------------------- Macros ----------------------------------*/ 82 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 83 84 /*--------------------------------- Xen IPIs ---------------------------------*/ 85 #ifdef SMP 86 struct xen_ipi_handler 87 { 88 driver_filter_t *filter; 89 const char *description; 90 }; 91 92 static struct xen_ipi_handler xen_ipis[] = 93 { 94 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 95 #ifdef __amd64__ 96 [IPI_TO_IDX(IPI_INVLOP)] = { xen_invlop, "itlb"}, 97 #else 98 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 99 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 100 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 101 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 102 #endif 103 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 104 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 105 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 106 [IPI_TO_IDX(IPI_SWI)] = { xen_ipi_swi_handler, "sw" }, 107 }; 108 #endif 109 110 /*------------------------------- Per-CPU Data -------------------------------*/ 111 #ifdef SMP 112 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 113 #endif 114 115 /*------------------------------- Xen PV APIC --------------------------------*/ 116 117 static void 118 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 119 { 120 #ifdef SMP 121 cpu_add(apic_id, boot_cpu); 122 #endif 123 } 124 125 static void 126 xen_pv_lapic_init(vm_paddr_t addr) 127 { 128 129 } 130 131 static void 132 xen_pv_lapic_setup(int boot) 133 { 134 135 } 136 137 static void 138 xen_pv_lapic_dump(const char *str) 139 { 140 141 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 142 } 143 144 static void 145 xen_pv_lapic_disable(void) 146 { 147 148 } 149 150 static bool 151 xen_pv_lapic_is_x2apic(void) 152 { 153 154 return (false); 155 } 156 157 static void 158 xen_pv_lapic_eoi(void) 159 { 160 161 XEN_APIC_UNSUPPORTED; 162 } 163 164 static int 165 xen_pv_lapic_id(void) 166 { 167 168 return (PCPU_GET(apic_id)); 169 } 170 171 static int 172 xen_pv_lapic_intr_pending(u_int vector) 173 { 174 175 XEN_APIC_UNSUPPORTED; 176 return (0); 177 } 178 179 static u_int 180 xen_pv_apic_cpuid(u_int apic_id) 181 { 182 #ifdef SMP 183 return (apic_cpuids[apic_id]); 184 #else 185 return (0); 186 #endif 187 } 188 189 static u_int 190 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 191 { 192 193 XEN_APIC_UNSUPPORTED; 194 return (0); 195 } 196 197 static u_int 198 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 199 { 200 201 XEN_APIC_UNSUPPORTED; 202 return (0); 203 } 204 205 static void 206 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 207 { 208 209 XEN_APIC_UNSUPPORTED; 210 } 211 212 static void 213 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 214 { 215 216 XEN_APIC_UNSUPPORTED; 217 } 218 219 static void 220 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 221 { 222 223 XEN_APIC_UNSUPPORTED; 224 } 225 226 static void 227 xen_pv_lapic_calibrate_timer(void) 228 { 229 230 } 231 232 static void 233 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 234 { 235 236 XEN_APIC_UNSUPPORTED; 237 } 238 239 static int 240 xen_pv_lapic_enable_pmc(void) 241 { 242 243 XEN_APIC_UNSUPPORTED; 244 return (0); 245 } 246 247 static void 248 xen_pv_lapic_disable_pmc(void) 249 { 250 251 XEN_APIC_UNSUPPORTED; 252 } 253 254 static void 255 xen_pv_lapic_reenable_pmc(void) 256 { 257 258 XEN_APIC_UNSUPPORTED; 259 } 260 261 static void 262 xen_pv_lapic_enable_cmc(void) 263 { 264 265 } 266 267 #ifdef SMP 268 static void 269 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 270 { 271 272 XEN_APIC_UNSUPPORTED; 273 } 274 275 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) 276 static void 277 send_nmi(int dest) 278 { 279 unsigned int cpu; 280 281 /* 282 * NMIs are not routed over event channels, and instead delivered as on 283 * native using the exception vector (#2). Triggering them can be done 284 * using the local APIC, or an hypercall as a shortcut like it's done 285 * below. 286 */ 287 switch(dest) { 288 case APIC_IPI_DEST_SELF: 289 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); 290 break; 291 case APIC_IPI_DEST_ALL: 292 CPU_FOREACH(cpu) 293 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 294 PCPU_ID_GET(cpu, vcpu_id), NULL); 295 break; 296 case APIC_IPI_DEST_OTHERS: 297 CPU_FOREACH(cpu) 298 if (cpu != PCPU_GET(cpuid)) 299 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 300 PCPU_ID_GET(cpu, vcpu_id), NULL); 301 break; 302 default: 303 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 304 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); 305 break; 306 } 307 } 308 #undef PCPU_ID_GET 309 310 static void 311 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 312 { 313 xen_intr_handle_t *ipi_handle; 314 int ipi_idx, to_cpu, self; 315 316 if (vector >= IPI_NMI_FIRST) { 317 send_nmi(dest); 318 return; 319 } 320 321 ipi_idx = IPI_TO_IDX(vector); 322 if (ipi_idx >= nitems(xen_ipis)) 323 panic("IPI out of range"); 324 325 switch(dest) { 326 case APIC_IPI_DEST_SELF: 327 ipi_handle = DPCPU_GET(ipi_handle); 328 xen_intr_signal(ipi_handle[ipi_idx]); 329 break; 330 case APIC_IPI_DEST_ALL: 331 CPU_FOREACH(to_cpu) { 332 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 333 xen_intr_signal(ipi_handle[ipi_idx]); 334 } 335 break; 336 case APIC_IPI_DEST_OTHERS: 337 self = PCPU_GET(cpuid); 338 CPU_FOREACH(to_cpu) { 339 if (to_cpu != self) { 340 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 341 xen_intr_signal(ipi_handle[ipi_idx]); 342 } 343 } 344 break; 345 default: 346 to_cpu = apic_cpuid(dest); 347 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 348 xen_intr_signal(ipi_handle[ipi_idx]); 349 break; 350 } 351 } 352 353 static int 354 xen_pv_lapic_ipi_wait(int delay) 355 { 356 357 XEN_APIC_UNSUPPORTED; 358 return (0); 359 } 360 #endif /* SMP */ 361 362 static int 363 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 364 { 365 366 XEN_APIC_UNSUPPORTED; 367 return (-1); 368 } 369 370 static void 371 xen_pv_lapic_ipi_free(int vector) 372 { 373 374 XEN_APIC_UNSUPPORTED; 375 } 376 377 static int 378 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 379 { 380 381 XEN_APIC_UNSUPPORTED; 382 return (0); 383 } 384 385 static int 386 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 387 { 388 389 XEN_APIC_UNSUPPORTED; 390 return (0); 391 } 392 393 static int 394 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 395 { 396 397 XEN_APIC_UNSUPPORTED; 398 return (0); 399 } 400 401 static int 402 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 403 enum intr_trigger trigger) 404 { 405 406 XEN_APIC_UNSUPPORTED; 407 return (0); 408 } 409 410 /* Xen apic_ops implementation */ 411 struct apic_ops xen_apic_ops = { 412 .create = xen_pv_lapic_create, 413 .init = xen_pv_lapic_init, 414 .xapic_mode = xen_pv_lapic_disable, 415 .is_x2apic = xen_pv_lapic_is_x2apic, 416 .setup = xen_pv_lapic_setup, 417 .dump = xen_pv_lapic_dump, 418 .disable = xen_pv_lapic_disable, 419 .eoi = xen_pv_lapic_eoi, 420 .id = xen_pv_lapic_id, 421 .intr_pending = xen_pv_lapic_intr_pending, 422 .set_logical_id = xen_pv_lapic_set_logical_id, 423 .cpuid = xen_pv_apic_cpuid, 424 .alloc_vector = xen_pv_apic_alloc_vector, 425 .alloc_vectors = xen_pv_apic_alloc_vectors, 426 .enable_vector = xen_pv_apic_enable_vector, 427 .disable_vector = xen_pv_apic_disable_vector, 428 .free_vector = xen_pv_apic_free_vector, 429 .calibrate_timer = xen_pv_lapic_calibrate_timer, 430 .enable_pmc = xen_pv_lapic_enable_pmc, 431 .disable_pmc = xen_pv_lapic_disable_pmc, 432 .reenable_pmc = xen_pv_lapic_reenable_pmc, 433 .enable_cmc = xen_pv_lapic_enable_cmc, 434 #ifdef SMP 435 .ipi_raw = xen_pv_lapic_ipi_raw, 436 .ipi_vectored = xen_pv_lapic_ipi_vectored, 437 .ipi_wait = xen_pv_lapic_ipi_wait, 438 #endif 439 .ipi_alloc = xen_pv_lapic_ipi_alloc, 440 .ipi_free = xen_pv_lapic_ipi_free, 441 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 442 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 443 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 444 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 445 }; 446 447 #ifdef SMP 448 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 449 /* 450 * These are C clones of the ASM functions found in apic_vector. 451 */ 452 static int 453 xen_ipi_bitmap_handler(void *arg) 454 { 455 struct trapframe *frame; 456 457 frame = arg; 458 ipi_bitmap_handler(*frame); 459 return (FILTER_HANDLED); 460 } 461 462 static int 463 xen_smp_rendezvous_action(void *arg) 464 { 465 #ifdef COUNT_IPIS 466 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 467 #endif /* COUNT_IPIS */ 468 469 smp_rendezvous_action(); 470 return (FILTER_HANDLED); 471 } 472 473 #ifdef __amd64__ 474 static int 475 xen_invlop(void *arg) 476 { 477 478 invlop_handler(); 479 return (FILTER_HANDLED); 480 } 481 482 #else /* __i386__ */ 483 484 static int 485 xen_invltlb(void *arg) 486 { 487 488 invltlb_handler(); 489 return (FILTER_HANDLED); 490 } 491 492 static int 493 xen_invlpg(void *arg) 494 { 495 496 invlpg_handler(); 497 return (FILTER_HANDLED); 498 } 499 500 static int 501 xen_invlrng(void *arg) 502 { 503 504 invlrng_handler(); 505 return (FILTER_HANDLED); 506 } 507 508 static int 509 xen_invlcache(void *arg) 510 { 511 512 invlcache_handler(); 513 return (FILTER_HANDLED); 514 } 515 #endif /* __amd64__ */ 516 517 static int 518 xen_cpustop_handler(void *arg) 519 { 520 521 cpustop_handler(); 522 return (FILTER_HANDLED); 523 } 524 525 static int 526 xen_cpususpend_handler(void *arg) 527 { 528 529 cpususpend_handler(); 530 return (FILTER_HANDLED); 531 } 532 533 static int 534 xen_ipi_swi_handler(void *arg) 535 { 536 struct trapframe *frame = arg; 537 538 ipi_swi_handler(*frame); 539 return (FILTER_HANDLED); 540 } 541 542 /*----------------------------- XEN PV IPI setup -----------------------------*/ 543 /* 544 * Those functions are provided outside of the Xen PV APIC implementation 545 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 546 * because on PVHVM there's an emulated LAPIC provided by Xen. 547 */ 548 static void 549 xen_cpu_ipi_init(int cpu) 550 { 551 xen_intr_handle_t *ipi_handle; 552 const struct xen_ipi_handler *ipi; 553 int idx, rc; 554 555 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 556 557 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 558 if (ipi->filter == NULL) { 559 ipi_handle[idx] = NULL; 560 continue; 561 } 562 563 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 564 INTR_TYPE_TTY, &ipi_handle[idx]); 565 if (rc != 0) 566 panic("Unable to allocate a XEN IPI port"); 567 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 568 } 569 } 570 571 static void 572 xen_setup_cpus(void) 573 { 574 int i; 575 576 if (!xen_vector_callback_enabled) 577 return; 578 579 CPU_FOREACH(i) 580 xen_cpu_ipi_init(i); 581 582 /* Set the xen pv ipi ops to replace the native ones */ 583 if (xen_hvm_domain()) 584 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 585 } 586 587 /* Switch to using PV IPIs as soon as the vcpu_id is set. */ 588 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL); 589 #endif /* SMP */ 590