1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/smp.h> 45 46 #include <x86/apicreg.h> 47 #include <x86/apicvar.h> 48 49 #include <xen/xen-os.h> 50 #include <xen/features.h> 51 #include <xen/gnttab.h> 52 #include <xen/hypervisor.h> 53 #include <xen/hvm.h> 54 #include <xen/xen_intr.h> 55 56 #include <xen/interface/vcpu.h> 57 58 /*--------------------------------- Macros -----------------------------------*/ 59 60 #define XEN_APIC_UNSUPPORTED \ 61 panic("%s: not available in Xen PV port.", __func__) 62 63 64 /*--------------------------- Forward Declarations ---------------------------*/ 65 #ifdef SMP 66 static driver_filter_t xen_smp_rendezvous_action; 67 static driver_filter_t xen_invltlb; 68 static driver_filter_t xen_invlpg; 69 static driver_filter_t xen_invlrng; 70 static driver_filter_t xen_invlcache; 71 #ifdef __i386__ 72 static driver_filter_t xen_lazypmap; 73 #endif 74 static driver_filter_t xen_ipi_bitmap_handler; 75 static driver_filter_t xen_cpustop_handler; 76 static driver_filter_t xen_cpususpend_handler; 77 static driver_filter_t xen_cpustophard_handler; 78 #endif 79 80 /*---------------------------- Extern Declarations ---------------------------*/ 81 /* Variables used by mp_machdep to perform the MMU related IPIs */ 82 #ifdef __i386__ 83 extern void pmap_lazyfix_action(void); 84 #endif 85 #ifdef __amd64__ 86 extern int pmap_pcid_enabled; 87 #endif 88 89 /*---------------------------------- Macros ----------------------------------*/ 90 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 91 92 /*--------------------------------- Xen IPIs ---------------------------------*/ 93 #ifdef SMP 94 struct xen_ipi_handler 95 { 96 driver_filter_t *filter; 97 const char *description; 98 }; 99 100 static struct xen_ipi_handler xen_ipis[] = 101 { 102 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 103 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 104 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 105 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 106 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 107 #ifdef __i386__ 108 [IPI_TO_IDX(IPI_LAZYPMAP)] = { xen_lazypmap, "lp" }, 109 #endif 110 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 111 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 112 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 113 [IPI_TO_IDX(IPI_STOP_HARD)] = { xen_cpustophard_handler, "sth" }, 114 }; 115 #endif 116 117 /*------------------------------- Per-CPU Data -------------------------------*/ 118 #ifdef SMP 119 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 120 #endif 121 122 /*------------------------------- Xen PV APIC --------------------------------*/ 123 124 static void 125 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 126 { 127 #ifdef SMP 128 cpu_add(apic_id, boot_cpu); 129 #endif 130 } 131 132 static void 133 xen_pv_lapic_init(vm_paddr_t addr) 134 { 135 136 } 137 138 static void 139 xen_pv_lapic_setup(int boot) 140 { 141 142 } 143 144 static void 145 xen_pv_lapic_dump(const char *str) 146 { 147 148 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 149 } 150 151 static void 152 xen_pv_lapic_disable(void) 153 { 154 155 } 156 157 static void 158 xen_pv_lapic_eoi(void) 159 { 160 161 XEN_APIC_UNSUPPORTED; 162 } 163 164 static int 165 xen_pv_lapic_id(void) 166 { 167 168 return (PCPU_GET(apic_id)); 169 } 170 171 static int 172 xen_pv_lapic_intr_pending(u_int vector) 173 { 174 175 XEN_APIC_UNSUPPORTED; 176 return (0); 177 } 178 179 static u_int 180 xen_pv_apic_cpuid(u_int apic_id) 181 { 182 #ifdef SMP 183 return (apic_cpuids[apic_id]); 184 #else 185 return (0); 186 #endif 187 } 188 189 static u_int 190 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 191 { 192 193 XEN_APIC_UNSUPPORTED; 194 return (0); 195 } 196 197 static u_int 198 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 199 { 200 201 XEN_APIC_UNSUPPORTED; 202 return (0); 203 } 204 205 static void 206 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 207 { 208 209 XEN_APIC_UNSUPPORTED; 210 } 211 212 static void 213 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 214 { 215 216 XEN_APIC_UNSUPPORTED; 217 } 218 219 static void 220 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 221 { 222 223 XEN_APIC_UNSUPPORTED; 224 } 225 226 static void 227 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 228 { 229 230 XEN_APIC_UNSUPPORTED; 231 } 232 233 static int 234 xen_pv_lapic_enable_pmc(void) 235 { 236 237 XEN_APIC_UNSUPPORTED; 238 return (0); 239 } 240 241 static void 242 xen_pv_lapic_disable_pmc(void) 243 { 244 245 XEN_APIC_UNSUPPORTED; 246 } 247 248 static void 249 xen_pv_lapic_reenable_pmc(void) 250 { 251 252 XEN_APIC_UNSUPPORTED; 253 } 254 255 static void 256 xen_pv_lapic_enable_cmc(void) 257 { 258 259 } 260 261 #ifdef SMP 262 static void 263 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 264 { 265 266 XEN_APIC_UNSUPPORTED; 267 } 268 269 static void 270 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 271 { 272 xen_intr_handle_t *ipi_handle; 273 int ipi_idx, to_cpu, self; 274 275 ipi_idx = IPI_TO_IDX(vector); 276 if (ipi_idx >= nitems(xen_ipis)) 277 panic("IPI out of range"); 278 279 switch(dest) { 280 case APIC_IPI_DEST_SELF: 281 ipi_handle = DPCPU_GET(ipi_handle); 282 xen_intr_signal(ipi_handle[ipi_idx]); 283 break; 284 case APIC_IPI_DEST_ALL: 285 CPU_FOREACH(to_cpu) { 286 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 287 xen_intr_signal(ipi_handle[ipi_idx]); 288 } 289 break; 290 case APIC_IPI_DEST_OTHERS: 291 self = PCPU_GET(cpuid); 292 CPU_FOREACH(to_cpu) { 293 if (to_cpu != self) { 294 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 295 xen_intr_signal(ipi_handle[ipi_idx]); 296 } 297 } 298 break; 299 default: 300 to_cpu = apic_cpuid(dest); 301 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 302 xen_intr_signal(ipi_handle[ipi_idx]); 303 break; 304 } 305 } 306 307 static int 308 xen_pv_lapic_ipi_wait(int delay) 309 { 310 311 XEN_APIC_UNSUPPORTED; 312 return (0); 313 } 314 315 static int 316 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 317 { 318 319 XEN_APIC_UNSUPPORTED; 320 return (-1); 321 } 322 323 static void 324 xen_pv_lapic_ipi_free(int vector) 325 { 326 327 XEN_APIC_UNSUPPORTED; 328 } 329 #endif /* SMP */ 330 331 static int 332 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 333 { 334 335 XEN_APIC_UNSUPPORTED; 336 return (0); 337 } 338 339 static int 340 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 341 { 342 343 XEN_APIC_UNSUPPORTED; 344 return (0); 345 } 346 347 static int 348 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 349 { 350 351 XEN_APIC_UNSUPPORTED; 352 return (0); 353 } 354 355 static int 356 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 357 enum intr_trigger trigger) 358 { 359 360 XEN_APIC_UNSUPPORTED; 361 return (0); 362 } 363 364 /* Xen apic_ops implementation */ 365 struct apic_ops xen_apic_ops = { 366 .create = xen_pv_lapic_create, 367 .init = xen_pv_lapic_init, 368 .xapic_mode = xen_pv_lapic_disable, 369 .setup = xen_pv_lapic_setup, 370 .dump = xen_pv_lapic_dump, 371 .disable = xen_pv_lapic_disable, 372 .eoi = xen_pv_lapic_eoi, 373 .id = xen_pv_lapic_id, 374 .intr_pending = xen_pv_lapic_intr_pending, 375 .set_logical_id = xen_pv_lapic_set_logical_id, 376 .cpuid = xen_pv_apic_cpuid, 377 .alloc_vector = xen_pv_apic_alloc_vector, 378 .alloc_vectors = xen_pv_apic_alloc_vectors, 379 .enable_vector = xen_pv_apic_enable_vector, 380 .disable_vector = xen_pv_apic_disable_vector, 381 .free_vector = xen_pv_apic_free_vector, 382 .enable_pmc = xen_pv_lapic_enable_pmc, 383 .disable_pmc = xen_pv_lapic_disable_pmc, 384 .reenable_pmc = xen_pv_lapic_reenable_pmc, 385 .enable_cmc = xen_pv_lapic_enable_cmc, 386 #ifdef SMP 387 .ipi_raw = xen_pv_lapic_ipi_raw, 388 .ipi_vectored = xen_pv_lapic_ipi_vectored, 389 .ipi_wait = xen_pv_lapic_ipi_wait, 390 .ipi_alloc = xen_pv_lapic_ipi_alloc, 391 .ipi_free = xen_pv_lapic_ipi_free, 392 #endif 393 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 394 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 395 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 396 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 397 }; 398 399 #ifdef SMP 400 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 401 /* 402 * These are C clones of the ASM functions found in apic_vector. 403 */ 404 static int 405 xen_ipi_bitmap_handler(void *arg) 406 { 407 struct trapframe *frame; 408 409 frame = arg; 410 ipi_bitmap_handler(*frame); 411 return (FILTER_HANDLED); 412 } 413 414 static int 415 xen_smp_rendezvous_action(void *arg) 416 { 417 #ifdef COUNT_IPIS 418 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 419 #endif /* COUNT_IPIS */ 420 421 smp_rendezvous_action(); 422 return (FILTER_HANDLED); 423 } 424 425 static int 426 xen_invltlb(void *arg) 427 { 428 429 invltlb_handler(); 430 return (FILTER_HANDLED); 431 } 432 433 #ifdef __amd64__ 434 static int 435 xen_invltlb_pcid(void *arg) 436 { 437 438 invltlb_pcid_handler(); 439 return (FILTER_HANDLED); 440 } 441 #endif 442 443 static int 444 xen_invlpg(void *arg) 445 { 446 447 invlpg_handler(); 448 return (FILTER_HANDLED); 449 } 450 451 #ifdef __amd64__ 452 static int 453 xen_invlpg_pcid(void *arg) 454 { 455 456 invlpg_pcid_handler(); 457 return (FILTER_HANDLED); 458 } 459 #endif 460 461 static int 462 xen_invlrng(void *arg) 463 { 464 465 invlrng_handler(); 466 return (FILTER_HANDLED); 467 } 468 469 static int 470 xen_invlcache(void *arg) 471 { 472 473 invlcache_handler(); 474 return (FILTER_HANDLED); 475 } 476 477 #ifdef __i386__ 478 static int 479 xen_lazypmap(void *arg) 480 { 481 482 pmap_lazyfix_action(); 483 return (FILTER_HANDLED); 484 } 485 #endif 486 487 static int 488 xen_cpustop_handler(void *arg) 489 { 490 491 cpustop_handler(); 492 return (FILTER_HANDLED); 493 } 494 495 static int 496 xen_cpususpend_handler(void *arg) 497 { 498 499 cpususpend_handler(); 500 return (FILTER_HANDLED); 501 } 502 503 static int 504 xen_cpustophard_handler(void *arg) 505 { 506 507 ipi_nmi_handler(); 508 return (FILTER_HANDLED); 509 } 510 511 /*----------------------------- XEN PV IPI setup -----------------------------*/ 512 /* 513 * Those functions are provided outside of the Xen PV APIC implementation 514 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 515 * because on PVHVM there's an emulated LAPIC provided by Xen. 516 */ 517 static void 518 xen_cpu_ipi_init(int cpu) 519 { 520 xen_intr_handle_t *ipi_handle; 521 const struct xen_ipi_handler *ipi; 522 device_t dev; 523 int idx, rc; 524 525 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 526 dev = pcpu_find(cpu)->pc_device; 527 KASSERT((dev != NULL), ("NULL pcpu device_t")); 528 529 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 530 531 if (ipi->filter == NULL) { 532 ipi_handle[idx] = NULL; 533 continue; 534 } 535 536 rc = xen_intr_alloc_and_bind_ipi(dev, cpu, ipi->filter, 537 INTR_TYPE_TTY, &ipi_handle[idx]); 538 if (rc != 0) 539 panic("Unable to allocate a XEN IPI port"); 540 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 541 } 542 } 543 544 static void 545 xen_setup_cpus(void) 546 { 547 int i; 548 549 if (!xen_vector_callback_enabled) 550 return; 551 552 #ifdef __amd64__ 553 if (pmap_pcid_enabled) { 554 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = xen_invltlb_pcid; 555 xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = xen_invlpg_pcid; 556 } 557 #endif 558 CPU_FOREACH(i) 559 xen_cpu_ipi_init(i); 560 561 /* Set the xen pv ipi ops to replace the native ones */ 562 if (xen_hvm_domain()) 563 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 564 } 565 566 /* We need to setup IPIs before APs are started */ 567 SYSINIT(xen_setup_cpus, SI_SUB_SMP-1, SI_ORDER_FIRST, xen_setup_cpus, NULL); 568 #endif /* SMP */ 569