1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/smp.h> 44 45 #include <x86/apicreg.h> 46 47 #include <xen/xen-os.h> 48 #include <xen/features.h> 49 #include <xen/gnttab.h> 50 #include <xen/hypervisor.h> 51 #include <xen/hvm.h> 52 #include <xen/xen_intr.h> 53 54 #include <xen/interface/vcpu.h> 55 56 /*--------------------------------- Macros -----------------------------------*/ 57 58 #define XEN_APIC_UNSUPPORTED \ 59 panic("%s: not available in Xen PV port.", __func__) 60 61 62 /*--------------------------- Forward Declarations ---------------------------*/ 63 #ifdef SMP 64 static driver_filter_t xen_smp_rendezvous_action; 65 static driver_filter_t xen_invltlb; 66 static driver_filter_t xen_invlpg; 67 static driver_filter_t xen_invlrng; 68 static driver_filter_t xen_invlcache; 69 #ifdef __i386__ 70 static driver_filter_t xen_lazypmap; 71 #endif 72 static driver_filter_t xen_ipi_bitmap_handler; 73 static driver_filter_t xen_cpustop_handler; 74 static driver_filter_t xen_cpususpend_handler; 75 static driver_filter_t xen_cpustophard_handler; 76 #endif 77 78 /*---------------------------- Extern Declarations ---------------------------*/ 79 /* Variables used by mp_machdep to perform the MMU related IPIs */ 80 #ifdef __i386__ 81 extern void pmap_lazyfix_action(void); 82 #endif 83 #ifdef __amd64__ 84 extern int pmap_pcid_enabled; 85 #endif 86 87 /*---------------------------------- Macros ----------------------------------*/ 88 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 89 90 /*--------------------------------- Xen IPIs ---------------------------------*/ 91 #ifdef SMP 92 struct xen_ipi_handler 93 { 94 driver_filter_t *filter; 95 const char *description; 96 }; 97 98 static struct xen_ipi_handler xen_ipis[] = 99 { 100 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 101 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 102 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 103 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 104 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 105 #ifdef __i386__ 106 [IPI_TO_IDX(IPI_LAZYPMAP)] = { xen_lazypmap, "lp" }, 107 #endif 108 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 109 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 110 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 111 [IPI_TO_IDX(IPI_STOP_HARD)] = { xen_cpustophard_handler, "sth" }, 112 }; 113 #endif 114 115 /*------------------------------- Per-CPU Data -------------------------------*/ 116 #ifdef SMP 117 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 118 #endif 119 120 /*------------------------------- Xen PV APIC --------------------------------*/ 121 122 static void 123 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 124 { 125 #ifdef SMP 126 cpu_add(apic_id, boot_cpu); 127 #endif 128 } 129 130 static void 131 xen_pv_lapic_init(vm_paddr_t addr) 132 { 133 134 } 135 136 static void 137 xen_pv_lapic_setup(int boot) 138 { 139 140 } 141 142 static void 143 xen_pv_lapic_dump(const char *str) 144 { 145 146 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 147 } 148 149 static void 150 xen_pv_lapic_disable(void) 151 { 152 153 } 154 155 static void 156 xen_pv_lapic_eoi(void) 157 { 158 159 XEN_APIC_UNSUPPORTED; 160 } 161 162 static int 163 xen_pv_lapic_id(void) 164 { 165 166 return (PCPU_GET(apic_id)); 167 } 168 169 static int 170 xen_pv_lapic_intr_pending(u_int vector) 171 { 172 173 XEN_APIC_UNSUPPORTED; 174 return (0); 175 } 176 177 static u_int 178 xen_pv_apic_cpuid(u_int apic_id) 179 { 180 #ifdef SMP 181 return (apic_cpuids[apic_id]); 182 #else 183 return (0); 184 #endif 185 } 186 187 static u_int 188 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 189 { 190 191 XEN_APIC_UNSUPPORTED; 192 return (0); 193 } 194 195 static u_int 196 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 197 { 198 199 XEN_APIC_UNSUPPORTED; 200 return (0); 201 } 202 203 static void 204 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 205 { 206 207 XEN_APIC_UNSUPPORTED; 208 } 209 210 static void 211 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 212 { 213 214 XEN_APIC_UNSUPPORTED; 215 } 216 217 static void 218 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 219 { 220 221 XEN_APIC_UNSUPPORTED; 222 } 223 224 static void 225 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 226 { 227 228 XEN_APIC_UNSUPPORTED; 229 } 230 231 static int 232 xen_pv_lapic_enable_pmc(void) 233 { 234 235 XEN_APIC_UNSUPPORTED; 236 return (0); 237 } 238 239 static void 240 xen_pv_lapic_disable_pmc(void) 241 { 242 243 XEN_APIC_UNSUPPORTED; 244 } 245 246 static void 247 xen_pv_lapic_reenable_pmc(void) 248 { 249 250 XEN_APIC_UNSUPPORTED; 251 } 252 253 static void 254 xen_pv_lapic_enable_cmc(void) 255 { 256 257 } 258 259 static void 260 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 261 { 262 263 XEN_APIC_UNSUPPORTED; 264 } 265 266 static void 267 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 268 { 269 xen_intr_handle_t *ipi_handle; 270 int ipi_idx, to_cpu, self; 271 272 ipi_idx = IPI_TO_IDX(vector); 273 if (ipi_idx >= nitems(xen_ipis)) 274 panic("IPI out of range"); 275 276 switch(dest) { 277 case APIC_IPI_DEST_SELF: 278 ipi_handle = DPCPU_GET(ipi_handle); 279 xen_intr_signal(ipi_handle[ipi_idx]); 280 break; 281 case APIC_IPI_DEST_ALL: 282 CPU_FOREACH(to_cpu) { 283 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 284 xen_intr_signal(ipi_handle[ipi_idx]); 285 } 286 break; 287 case APIC_IPI_DEST_OTHERS: 288 self = PCPU_GET(cpuid); 289 CPU_FOREACH(to_cpu) { 290 if (to_cpu != self) { 291 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 292 xen_intr_signal(ipi_handle[ipi_idx]); 293 } 294 } 295 break; 296 default: 297 to_cpu = apic_cpuid(dest); 298 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 299 xen_intr_signal(ipi_handle[ipi_idx]); 300 break; 301 } 302 } 303 304 static int 305 xen_pv_lapic_ipi_wait(int delay) 306 { 307 308 XEN_APIC_UNSUPPORTED; 309 return (0); 310 } 311 312 static int 313 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 314 { 315 316 XEN_APIC_UNSUPPORTED; 317 return (0); 318 } 319 320 static int 321 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 322 { 323 324 XEN_APIC_UNSUPPORTED; 325 return (0); 326 } 327 328 static int 329 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 330 { 331 332 XEN_APIC_UNSUPPORTED; 333 return (0); 334 } 335 336 static int 337 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 338 enum intr_trigger trigger) 339 { 340 341 XEN_APIC_UNSUPPORTED; 342 return (0); 343 } 344 345 /* Xen apic_ops implementation */ 346 struct apic_ops xen_apic_ops = { 347 .create = xen_pv_lapic_create, 348 .init = xen_pv_lapic_init, 349 .setup = xen_pv_lapic_setup, 350 .dump = xen_pv_lapic_dump, 351 .disable = xen_pv_lapic_disable, 352 .eoi = xen_pv_lapic_eoi, 353 .id = xen_pv_lapic_id, 354 .intr_pending = xen_pv_lapic_intr_pending, 355 .set_logical_id = xen_pv_lapic_set_logical_id, 356 .cpuid = xen_pv_apic_cpuid, 357 .alloc_vector = xen_pv_apic_alloc_vector, 358 .alloc_vectors = xen_pv_apic_alloc_vectors, 359 .enable_vector = xen_pv_apic_enable_vector, 360 .disable_vector = xen_pv_apic_disable_vector, 361 .free_vector = xen_pv_apic_free_vector, 362 .enable_pmc = xen_pv_lapic_enable_pmc, 363 .disable_pmc = xen_pv_lapic_disable_pmc, 364 .reenable_pmc = xen_pv_lapic_reenable_pmc, 365 .enable_cmc = xen_pv_lapic_enable_cmc, 366 .ipi_raw = xen_pv_lapic_ipi_raw, 367 .ipi_vectored = xen_pv_lapic_ipi_vectored, 368 .ipi_wait = xen_pv_lapic_ipi_wait, 369 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 370 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 371 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 372 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 373 }; 374 375 #ifdef SMP 376 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 377 /* 378 * These are C clones of the ASM functions found in apic_vector. 379 */ 380 static int 381 xen_ipi_bitmap_handler(void *arg) 382 { 383 struct trapframe *frame; 384 385 frame = arg; 386 ipi_bitmap_handler(*frame); 387 return (FILTER_HANDLED); 388 } 389 390 static int 391 xen_smp_rendezvous_action(void *arg) 392 { 393 #ifdef COUNT_IPIS 394 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 395 #endif /* COUNT_IPIS */ 396 397 smp_rendezvous_action(); 398 return (FILTER_HANDLED); 399 } 400 401 static int 402 xen_invltlb(void *arg) 403 { 404 405 invltlb_handler(); 406 return (FILTER_HANDLED); 407 } 408 409 #ifdef __amd64__ 410 static int 411 xen_invltlb_pcid(void *arg) 412 { 413 414 invltlb_pcid_handler(); 415 return (FILTER_HANDLED); 416 } 417 #endif 418 419 static int 420 xen_invlpg(void *arg) 421 { 422 423 invlpg_handler(); 424 return (FILTER_HANDLED); 425 } 426 427 #ifdef __amd64__ 428 static int 429 xen_invlpg_pcid(void *arg) 430 { 431 432 invlpg_pcid_handler(); 433 return (FILTER_HANDLED); 434 } 435 #endif 436 437 static int 438 xen_invlrng(void *arg) 439 { 440 441 invlrng_handler(); 442 return (FILTER_HANDLED); 443 } 444 445 static int 446 xen_invlcache(void *arg) 447 { 448 449 invlcache_handler(); 450 return (FILTER_HANDLED); 451 } 452 453 #ifdef __i386__ 454 static int 455 xen_lazypmap(void *arg) 456 { 457 458 pmap_lazyfix_action(); 459 return (FILTER_HANDLED); 460 } 461 #endif 462 463 static int 464 xen_cpustop_handler(void *arg) 465 { 466 467 cpustop_handler(); 468 return (FILTER_HANDLED); 469 } 470 471 static int 472 xen_cpususpend_handler(void *arg) 473 { 474 475 cpususpend_handler(); 476 return (FILTER_HANDLED); 477 } 478 479 static int 480 xen_cpustophard_handler(void *arg) 481 { 482 483 ipi_nmi_handler(); 484 return (FILTER_HANDLED); 485 } 486 487 /*----------------------------- XEN PV IPI setup -----------------------------*/ 488 /* 489 * Those functions are provided outside of the Xen PV APIC implementation 490 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 491 * because on PVHVM there's an emulated LAPIC provided by Xen. 492 */ 493 static void 494 xen_cpu_ipi_init(int cpu) 495 { 496 xen_intr_handle_t *ipi_handle; 497 const struct xen_ipi_handler *ipi; 498 device_t dev; 499 int idx, rc; 500 501 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 502 dev = pcpu_find(cpu)->pc_device; 503 KASSERT((dev != NULL), ("NULL pcpu device_t")); 504 505 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 506 507 if (ipi->filter == NULL) { 508 ipi_handle[idx] = NULL; 509 continue; 510 } 511 512 rc = xen_intr_alloc_and_bind_ipi(dev, cpu, ipi->filter, 513 INTR_TYPE_TTY, &ipi_handle[idx]); 514 if (rc != 0) 515 panic("Unable to allocate a XEN IPI port"); 516 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 517 } 518 } 519 520 static void 521 xen_setup_cpus(void) 522 { 523 int i; 524 525 if (!xen_vector_callback_enabled) 526 return; 527 528 #ifdef __amd64__ 529 if (pmap_pcid_enabled) { 530 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = xen_invltlb_pcid; 531 xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = xen_invlpg_pcid; 532 } 533 #endif 534 CPU_FOREACH(i) 535 xen_cpu_ipi_init(i); 536 537 /* Set the xen pv ipi ops to replace the native ones */ 538 if (xen_hvm_domain()) 539 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 540 } 541 542 /* We need to setup IPIs before APs are started */ 543 SYSINIT(xen_setup_cpus, SI_SUB_SMP-1, SI_ORDER_FIRST, xen_setup_cpus, NULL); 544 #endif /* SMP */ 545