1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/smp.h> 45 46 #include <x86/apicreg.h> 47 #include <x86/apicvar.h> 48 49 #include <xen/xen-os.h> 50 #include <xen/features.h> 51 #include <xen/gnttab.h> 52 #include <xen/hypervisor.h> 53 #include <xen/hvm.h> 54 #include <xen/xen_intr.h> 55 56 #include <xen/interface/vcpu.h> 57 58 /*--------------------------------- Macros -----------------------------------*/ 59 60 #define XEN_APIC_UNSUPPORTED \ 61 panic("%s: not available in Xen PV port.", __func__) 62 63 64 /*--------------------------- Forward Declarations ---------------------------*/ 65 #ifdef SMP 66 static driver_filter_t xen_smp_rendezvous_action; 67 static driver_filter_t xen_invltlb; 68 static driver_filter_t xen_invlpg; 69 static driver_filter_t xen_invlrng; 70 static driver_filter_t xen_invlcache; 71 static driver_filter_t xen_ipi_bitmap_handler; 72 static driver_filter_t xen_cpustop_handler; 73 static driver_filter_t xen_cpususpend_handler; 74 static driver_filter_t xen_cpustophard_handler; 75 #endif 76 77 /*---------------------------- Extern Declarations ---------------------------*/ 78 /* Variables used by mp_machdep to perform the MMU related IPIs */ 79 #ifdef __amd64__ 80 extern int pmap_pcid_enabled; 81 #endif 82 83 /*---------------------------------- Macros ----------------------------------*/ 84 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 85 86 /*--------------------------------- Xen IPIs ---------------------------------*/ 87 #ifdef SMP 88 struct xen_ipi_handler 89 { 90 driver_filter_t *filter; 91 const char *description; 92 }; 93 94 static struct xen_ipi_handler xen_ipis[] = 95 { 96 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 97 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 98 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 99 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 100 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 101 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 102 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 103 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 104 [IPI_TO_IDX(IPI_STOP_HARD)] = { xen_cpustophard_handler, "sth" }, 105 }; 106 #endif 107 108 /*------------------------------- Per-CPU Data -------------------------------*/ 109 #ifdef SMP 110 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 111 #endif 112 113 /*------------------------------- Xen PV APIC --------------------------------*/ 114 115 static void 116 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 117 { 118 #ifdef SMP 119 cpu_add(apic_id, boot_cpu); 120 #endif 121 } 122 123 static void 124 xen_pv_lapic_init(vm_paddr_t addr) 125 { 126 127 } 128 129 static void 130 xen_pv_lapic_setup(int boot) 131 { 132 133 } 134 135 static void 136 xen_pv_lapic_dump(const char *str) 137 { 138 139 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 140 } 141 142 static void 143 xen_pv_lapic_disable(void) 144 { 145 146 } 147 148 static void 149 xen_pv_lapic_eoi(void) 150 { 151 152 XEN_APIC_UNSUPPORTED; 153 } 154 155 static int 156 xen_pv_lapic_id(void) 157 { 158 159 return (PCPU_GET(apic_id)); 160 } 161 162 static int 163 xen_pv_lapic_intr_pending(u_int vector) 164 { 165 166 XEN_APIC_UNSUPPORTED; 167 return (0); 168 } 169 170 static u_int 171 xen_pv_apic_cpuid(u_int apic_id) 172 { 173 #ifdef SMP 174 return (apic_cpuids[apic_id]); 175 #else 176 return (0); 177 #endif 178 } 179 180 static u_int 181 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 182 { 183 184 XEN_APIC_UNSUPPORTED; 185 return (0); 186 } 187 188 static u_int 189 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 190 { 191 192 XEN_APIC_UNSUPPORTED; 193 return (0); 194 } 195 196 static void 197 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 198 { 199 200 XEN_APIC_UNSUPPORTED; 201 } 202 203 static void 204 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 205 { 206 207 XEN_APIC_UNSUPPORTED; 208 } 209 210 static void 211 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 212 { 213 214 XEN_APIC_UNSUPPORTED; 215 } 216 217 static void 218 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 219 { 220 221 XEN_APIC_UNSUPPORTED; 222 } 223 224 static int 225 xen_pv_lapic_enable_pmc(void) 226 { 227 228 XEN_APIC_UNSUPPORTED; 229 return (0); 230 } 231 232 static void 233 xen_pv_lapic_disable_pmc(void) 234 { 235 236 XEN_APIC_UNSUPPORTED; 237 } 238 239 static void 240 xen_pv_lapic_reenable_pmc(void) 241 { 242 243 XEN_APIC_UNSUPPORTED; 244 } 245 246 static void 247 xen_pv_lapic_enable_cmc(void) 248 { 249 250 } 251 252 #ifdef SMP 253 static void 254 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 255 { 256 257 XEN_APIC_UNSUPPORTED; 258 } 259 260 static void 261 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 262 { 263 xen_intr_handle_t *ipi_handle; 264 int ipi_idx, to_cpu, self; 265 266 ipi_idx = IPI_TO_IDX(vector); 267 if (ipi_idx >= nitems(xen_ipis)) 268 panic("IPI out of range"); 269 270 switch(dest) { 271 case APIC_IPI_DEST_SELF: 272 ipi_handle = DPCPU_GET(ipi_handle); 273 xen_intr_signal(ipi_handle[ipi_idx]); 274 break; 275 case APIC_IPI_DEST_ALL: 276 CPU_FOREACH(to_cpu) { 277 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 278 xen_intr_signal(ipi_handle[ipi_idx]); 279 } 280 break; 281 case APIC_IPI_DEST_OTHERS: 282 self = PCPU_GET(cpuid); 283 CPU_FOREACH(to_cpu) { 284 if (to_cpu != self) { 285 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 286 xen_intr_signal(ipi_handle[ipi_idx]); 287 } 288 } 289 break; 290 default: 291 to_cpu = apic_cpuid(dest); 292 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 293 xen_intr_signal(ipi_handle[ipi_idx]); 294 break; 295 } 296 } 297 298 static int 299 xen_pv_lapic_ipi_wait(int delay) 300 { 301 302 XEN_APIC_UNSUPPORTED; 303 return (0); 304 } 305 306 static int 307 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 308 { 309 310 XEN_APIC_UNSUPPORTED; 311 return (-1); 312 } 313 314 static void 315 xen_pv_lapic_ipi_free(int vector) 316 { 317 318 XEN_APIC_UNSUPPORTED; 319 } 320 #endif /* SMP */ 321 322 static int 323 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 324 { 325 326 XEN_APIC_UNSUPPORTED; 327 return (0); 328 } 329 330 static int 331 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 332 { 333 334 XEN_APIC_UNSUPPORTED; 335 return (0); 336 } 337 338 static int 339 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 340 { 341 342 XEN_APIC_UNSUPPORTED; 343 return (0); 344 } 345 346 static int 347 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 348 enum intr_trigger trigger) 349 { 350 351 XEN_APIC_UNSUPPORTED; 352 return (0); 353 } 354 355 /* Xen apic_ops implementation */ 356 struct apic_ops xen_apic_ops = { 357 .create = xen_pv_lapic_create, 358 .init = xen_pv_lapic_init, 359 .xapic_mode = xen_pv_lapic_disable, 360 .setup = xen_pv_lapic_setup, 361 .dump = xen_pv_lapic_dump, 362 .disable = xen_pv_lapic_disable, 363 .eoi = xen_pv_lapic_eoi, 364 .id = xen_pv_lapic_id, 365 .intr_pending = xen_pv_lapic_intr_pending, 366 .set_logical_id = xen_pv_lapic_set_logical_id, 367 .cpuid = xen_pv_apic_cpuid, 368 .alloc_vector = xen_pv_apic_alloc_vector, 369 .alloc_vectors = xen_pv_apic_alloc_vectors, 370 .enable_vector = xen_pv_apic_enable_vector, 371 .disable_vector = xen_pv_apic_disable_vector, 372 .free_vector = xen_pv_apic_free_vector, 373 .enable_pmc = xen_pv_lapic_enable_pmc, 374 .disable_pmc = xen_pv_lapic_disable_pmc, 375 .reenable_pmc = xen_pv_lapic_reenable_pmc, 376 .enable_cmc = xen_pv_lapic_enable_cmc, 377 #ifdef SMP 378 .ipi_raw = xen_pv_lapic_ipi_raw, 379 .ipi_vectored = xen_pv_lapic_ipi_vectored, 380 .ipi_wait = xen_pv_lapic_ipi_wait, 381 .ipi_alloc = xen_pv_lapic_ipi_alloc, 382 .ipi_free = xen_pv_lapic_ipi_free, 383 #endif 384 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 385 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 386 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 387 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 388 }; 389 390 #ifdef SMP 391 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 392 /* 393 * These are C clones of the ASM functions found in apic_vector. 394 */ 395 static int 396 xen_ipi_bitmap_handler(void *arg) 397 { 398 struct trapframe *frame; 399 400 frame = arg; 401 ipi_bitmap_handler(*frame); 402 return (FILTER_HANDLED); 403 } 404 405 static int 406 xen_smp_rendezvous_action(void *arg) 407 { 408 #ifdef COUNT_IPIS 409 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 410 #endif /* COUNT_IPIS */ 411 412 smp_rendezvous_action(); 413 return (FILTER_HANDLED); 414 } 415 416 static int 417 xen_invltlb(void *arg) 418 { 419 420 invltlb_handler(); 421 return (FILTER_HANDLED); 422 } 423 424 #ifdef __amd64__ 425 static int 426 xen_invltlb_invpcid(void *arg) 427 { 428 429 invltlb_invpcid_handler(); 430 return (FILTER_HANDLED); 431 } 432 433 static int 434 xen_invltlb_pcid(void *arg) 435 { 436 437 invltlb_pcid_handler(); 438 return (FILTER_HANDLED); 439 } 440 #endif 441 442 static int 443 xen_invlpg(void *arg) 444 { 445 446 invlpg_handler(); 447 return (FILTER_HANDLED); 448 } 449 450 static int 451 xen_invlrng(void *arg) 452 { 453 454 invlrng_handler(); 455 return (FILTER_HANDLED); 456 } 457 458 static int 459 xen_invlcache(void *arg) 460 { 461 462 invlcache_handler(); 463 return (FILTER_HANDLED); 464 } 465 466 static int 467 xen_cpustop_handler(void *arg) 468 { 469 470 cpustop_handler(); 471 return (FILTER_HANDLED); 472 } 473 474 static int 475 xen_cpususpend_handler(void *arg) 476 { 477 478 cpususpend_handler(); 479 return (FILTER_HANDLED); 480 } 481 482 static int 483 xen_cpustophard_handler(void *arg) 484 { 485 486 ipi_nmi_handler(); 487 return (FILTER_HANDLED); 488 } 489 490 /*----------------------------- XEN PV IPI setup -----------------------------*/ 491 /* 492 * Those functions are provided outside of the Xen PV APIC implementation 493 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 494 * because on PVHVM there's an emulated LAPIC provided by Xen. 495 */ 496 static void 497 xen_cpu_ipi_init(int cpu) 498 { 499 xen_intr_handle_t *ipi_handle; 500 const struct xen_ipi_handler *ipi; 501 device_t dev; 502 int idx, rc; 503 504 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 505 dev = pcpu_find(cpu)->pc_device; 506 KASSERT((dev != NULL), ("NULL pcpu device_t")); 507 508 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 509 510 if (ipi->filter == NULL) { 511 ipi_handle[idx] = NULL; 512 continue; 513 } 514 515 rc = xen_intr_alloc_and_bind_ipi(dev, cpu, ipi->filter, 516 INTR_TYPE_TTY, &ipi_handle[idx]); 517 if (rc != 0) 518 panic("Unable to allocate a XEN IPI port"); 519 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 520 } 521 } 522 523 static void 524 xen_setup_cpus(void) 525 { 526 int i; 527 528 if (!xen_vector_callback_enabled) 529 return; 530 531 #ifdef __amd64__ 532 if (pmap_pcid_enabled) { 533 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = invpcid_works ? 534 xen_invltlb_invpcid : xen_invltlb_pcid; 535 } 536 #endif 537 CPU_FOREACH(i) 538 xen_cpu_ipi_init(i); 539 540 /* Set the xen pv ipi ops to replace the native ones */ 541 if (xen_hvm_domain()) 542 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 543 } 544 545 /* We need to setup IPIs before APs are started */ 546 SYSINIT(xen_setup_cpus, SI_SUB_SMP-1, SI_ORDER_FIRST, xen_setup_cpus, NULL); 547 #endif /* SMP */ 548