1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/md_var.h> 45 #include <machine/smp.h> 46 47 #include <x86/apicreg.h> 48 #include <x86/apicvar.h> 49 50 #include <xen/xen-os.h> 51 #include <xen/features.h> 52 #include <xen/gnttab.h> 53 #include <xen/hypervisor.h> 54 #include <xen/hvm.h> 55 #include <xen/xen_intr.h> 56 57 #include <xen/interface/vcpu.h> 58 59 /*--------------------------------- Macros -----------------------------------*/ 60 61 #define XEN_APIC_UNSUPPORTED \ 62 panic("%s: not available in Xen PV port.", __func__) 63 64 65 /*--------------------------- Forward Declarations ---------------------------*/ 66 #ifdef SMP 67 static driver_filter_t xen_smp_rendezvous_action; 68 #ifdef __amd64__ 69 static driver_filter_t xen_invlop; 70 #else 71 static driver_filter_t xen_invltlb; 72 static driver_filter_t xen_invlpg; 73 static driver_filter_t xen_invlrng; 74 static driver_filter_t xen_invlcache; 75 #endif 76 static driver_filter_t xen_ipi_bitmap_handler; 77 static driver_filter_t xen_cpustop_handler; 78 static driver_filter_t xen_cpususpend_handler; 79 #endif 80 81 /*---------------------------------- Macros ----------------------------------*/ 82 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 83 84 /*--------------------------------- Xen IPIs ---------------------------------*/ 85 #ifdef SMP 86 struct xen_ipi_handler 87 { 88 driver_filter_t *filter; 89 const char *description; 90 }; 91 92 static struct xen_ipi_handler xen_ipis[] = 93 { 94 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 95 #ifdef __amd64__ 96 [IPI_TO_IDX(IPI_INVLOP)] = { xen_invlop, "itlb"}, 97 #else 98 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 99 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 100 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 101 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 102 #endif 103 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 104 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 105 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 106 }; 107 #endif 108 109 /*------------------------------- Per-CPU Data -------------------------------*/ 110 #ifdef SMP 111 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 112 #endif 113 114 /*------------------------------- Xen PV APIC --------------------------------*/ 115 116 static void 117 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 118 { 119 #ifdef SMP 120 cpu_add(apic_id, boot_cpu); 121 #endif 122 } 123 124 static void 125 xen_pv_lapic_init(vm_paddr_t addr) 126 { 127 128 } 129 130 static void 131 xen_pv_lapic_setup(int boot) 132 { 133 134 } 135 136 static void 137 xen_pv_lapic_dump(const char *str) 138 { 139 140 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 141 } 142 143 static void 144 xen_pv_lapic_disable(void) 145 { 146 147 } 148 149 static bool 150 xen_pv_lapic_is_x2apic(void) 151 { 152 153 return (false); 154 } 155 156 static void 157 xen_pv_lapic_eoi(void) 158 { 159 160 XEN_APIC_UNSUPPORTED; 161 } 162 163 static int 164 xen_pv_lapic_id(void) 165 { 166 167 return (PCPU_GET(apic_id)); 168 } 169 170 static int 171 xen_pv_lapic_intr_pending(u_int vector) 172 { 173 174 XEN_APIC_UNSUPPORTED; 175 return (0); 176 } 177 178 static u_int 179 xen_pv_apic_cpuid(u_int apic_id) 180 { 181 #ifdef SMP 182 return (apic_cpuids[apic_id]); 183 #else 184 return (0); 185 #endif 186 } 187 188 static u_int 189 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 190 { 191 192 XEN_APIC_UNSUPPORTED; 193 return (0); 194 } 195 196 static u_int 197 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 198 { 199 200 XEN_APIC_UNSUPPORTED; 201 return (0); 202 } 203 204 static void 205 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 206 { 207 208 XEN_APIC_UNSUPPORTED; 209 } 210 211 static void 212 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 213 { 214 215 XEN_APIC_UNSUPPORTED; 216 } 217 218 static void 219 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 220 { 221 222 XEN_APIC_UNSUPPORTED; 223 } 224 225 static void 226 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 227 { 228 229 XEN_APIC_UNSUPPORTED; 230 } 231 232 static int 233 xen_pv_lapic_enable_pmc(void) 234 { 235 236 XEN_APIC_UNSUPPORTED; 237 return (0); 238 } 239 240 static void 241 xen_pv_lapic_disable_pmc(void) 242 { 243 244 XEN_APIC_UNSUPPORTED; 245 } 246 247 static void 248 xen_pv_lapic_reenable_pmc(void) 249 { 250 251 XEN_APIC_UNSUPPORTED; 252 } 253 254 static void 255 xen_pv_lapic_enable_cmc(void) 256 { 257 258 } 259 260 #ifdef SMP 261 static void 262 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 263 { 264 265 XEN_APIC_UNSUPPORTED; 266 } 267 268 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) 269 static void 270 send_nmi(int dest) 271 { 272 unsigned int cpu; 273 274 /* 275 * NMIs are not routed over event channels, and instead delivered as on 276 * native using the exception vector (#2). Triggering them can be done 277 * using the local APIC, or an hypercall as a shortcut like it's done 278 * below. 279 */ 280 switch(dest) { 281 case APIC_IPI_DEST_SELF: 282 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); 283 break; 284 case APIC_IPI_DEST_ALL: 285 CPU_FOREACH(cpu) 286 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 287 PCPU_ID_GET(cpu, vcpu_id), NULL); 288 break; 289 case APIC_IPI_DEST_OTHERS: 290 CPU_FOREACH(cpu) 291 if (cpu != PCPU_GET(cpuid)) 292 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 293 PCPU_ID_GET(cpu, vcpu_id), NULL); 294 break; 295 default: 296 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 297 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); 298 break; 299 } 300 } 301 #undef PCPU_ID_GET 302 303 static void 304 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 305 { 306 xen_intr_handle_t *ipi_handle; 307 int ipi_idx, to_cpu, self; 308 309 if (vector >= IPI_NMI_FIRST) { 310 send_nmi(dest); 311 return; 312 } 313 314 ipi_idx = IPI_TO_IDX(vector); 315 if (ipi_idx >= nitems(xen_ipis)) 316 panic("IPI out of range"); 317 318 switch(dest) { 319 case APIC_IPI_DEST_SELF: 320 ipi_handle = DPCPU_GET(ipi_handle); 321 xen_intr_signal(ipi_handle[ipi_idx]); 322 break; 323 case APIC_IPI_DEST_ALL: 324 CPU_FOREACH(to_cpu) { 325 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 326 xen_intr_signal(ipi_handle[ipi_idx]); 327 } 328 break; 329 case APIC_IPI_DEST_OTHERS: 330 self = PCPU_GET(cpuid); 331 CPU_FOREACH(to_cpu) { 332 if (to_cpu != self) { 333 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 334 xen_intr_signal(ipi_handle[ipi_idx]); 335 } 336 } 337 break; 338 default: 339 to_cpu = apic_cpuid(dest); 340 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 341 xen_intr_signal(ipi_handle[ipi_idx]); 342 break; 343 } 344 } 345 346 static int 347 xen_pv_lapic_ipi_wait(int delay) 348 { 349 350 XEN_APIC_UNSUPPORTED; 351 return (0); 352 } 353 #endif /* SMP */ 354 355 static int 356 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 357 { 358 359 XEN_APIC_UNSUPPORTED; 360 return (-1); 361 } 362 363 static void 364 xen_pv_lapic_ipi_free(int vector) 365 { 366 367 XEN_APIC_UNSUPPORTED; 368 } 369 370 static int 371 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 372 { 373 374 XEN_APIC_UNSUPPORTED; 375 return (0); 376 } 377 378 static int 379 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 380 { 381 382 XEN_APIC_UNSUPPORTED; 383 return (0); 384 } 385 386 static int 387 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 388 { 389 390 XEN_APIC_UNSUPPORTED; 391 return (0); 392 } 393 394 static int 395 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 396 enum intr_trigger trigger) 397 { 398 399 XEN_APIC_UNSUPPORTED; 400 return (0); 401 } 402 403 /* Xen apic_ops implementation */ 404 struct apic_ops xen_apic_ops = { 405 .create = xen_pv_lapic_create, 406 .init = xen_pv_lapic_init, 407 .xapic_mode = xen_pv_lapic_disable, 408 .is_x2apic = xen_pv_lapic_is_x2apic, 409 .setup = xen_pv_lapic_setup, 410 .dump = xen_pv_lapic_dump, 411 .disable = xen_pv_lapic_disable, 412 .eoi = xen_pv_lapic_eoi, 413 .id = xen_pv_lapic_id, 414 .intr_pending = xen_pv_lapic_intr_pending, 415 .set_logical_id = xen_pv_lapic_set_logical_id, 416 .cpuid = xen_pv_apic_cpuid, 417 .alloc_vector = xen_pv_apic_alloc_vector, 418 .alloc_vectors = xen_pv_apic_alloc_vectors, 419 .enable_vector = xen_pv_apic_enable_vector, 420 .disable_vector = xen_pv_apic_disable_vector, 421 .free_vector = xen_pv_apic_free_vector, 422 .enable_pmc = xen_pv_lapic_enable_pmc, 423 .disable_pmc = xen_pv_lapic_disable_pmc, 424 .reenable_pmc = xen_pv_lapic_reenable_pmc, 425 .enable_cmc = xen_pv_lapic_enable_cmc, 426 #ifdef SMP 427 .ipi_raw = xen_pv_lapic_ipi_raw, 428 .ipi_vectored = xen_pv_lapic_ipi_vectored, 429 .ipi_wait = xen_pv_lapic_ipi_wait, 430 #endif 431 .ipi_alloc = xen_pv_lapic_ipi_alloc, 432 .ipi_free = xen_pv_lapic_ipi_free, 433 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 434 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 435 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 436 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 437 }; 438 439 #ifdef SMP 440 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 441 /* 442 * These are C clones of the ASM functions found in apic_vector. 443 */ 444 static int 445 xen_ipi_bitmap_handler(void *arg) 446 { 447 struct trapframe *frame; 448 449 frame = arg; 450 ipi_bitmap_handler(*frame); 451 return (FILTER_HANDLED); 452 } 453 454 static int 455 xen_smp_rendezvous_action(void *arg) 456 { 457 #ifdef COUNT_IPIS 458 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 459 #endif /* COUNT_IPIS */ 460 461 smp_rendezvous_action(); 462 return (FILTER_HANDLED); 463 } 464 465 #ifdef __amd64__ 466 static int 467 xen_invlop(void *arg) 468 { 469 470 invlop_handler(); 471 return (FILTER_HANDLED); 472 } 473 474 #else /* __i386__ */ 475 476 static int 477 xen_invltlb(void *arg) 478 { 479 480 invltlb_handler(); 481 return (FILTER_HANDLED); 482 } 483 484 static int 485 xen_invlpg(void *arg) 486 { 487 488 invlpg_handler(); 489 return (FILTER_HANDLED); 490 } 491 492 static int 493 xen_invlrng(void *arg) 494 { 495 496 invlrng_handler(); 497 return (FILTER_HANDLED); 498 } 499 500 static int 501 xen_invlcache(void *arg) 502 { 503 504 invlcache_handler(); 505 return (FILTER_HANDLED); 506 } 507 #endif /* __amd64__ */ 508 509 static int 510 xen_cpustop_handler(void *arg) 511 { 512 513 cpustop_handler(); 514 return (FILTER_HANDLED); 515 } 516 517 static int 518 xen_cpususpend_handler(void *arg) 519 { 520 521 cpususpend_handler(); 522 return (FILTER_HANDLED); 523 } 524 525 /*----------------------------- XEN PV IPI setup -----------------------------*/ 526 /* 527 * Those functions are provided outside of the Xen PV APIC implementation 528 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 529 * because on PVHVM there's an emulated LAPIC provided by Xen. 530 */ 531 static void 532 xen_cpu_ipi_init(int cpu) 533 { 534 xen_intr_handle_t *ipi_handle; 535 const struct xen_ipi_handler *ipi; 536 int idx, rc; 537 538 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 539 540 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 541 542 if (ipi->filter == NULL) { 543 ipi_handle[idx] = NULL; 544 continue; 545 } 546 547 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 548 INTR_TYPE_TTY, &ipi_handle[idx]); 549 if (rc != 0) 550 panic("Unable to allocate a XEN IPI port"); 551 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 552 } 553 } 554 555 static void 556 xen_setup_cpus(void) 557 { 558 int i; 559 560 if (!xen_vector_callback_enabled) 561 return; 562 563 CPU_FOREACH(i) 564 xen_cpu_ipi_init(i); 565 566 /* Set the xen pv ipi ops to replace the native ones */ 567 if (xen_hvm_domain()) 568 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 569 } 570 571 /* Switch to using PV IPIs as soon as the vcpu_id is set. */ 572 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL); 573 #endif /* SMP */ 574