1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright 2011 Nathan Whitehorn 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_platform.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/module.h> 36 #include <sys/bus.h> 37 #include <sys/conf.h> 38 #include <sys/kernel.h> 39 #include <sys/lock.h> 40 #include <sys/malloc.h> 41 #include <sys/mutex.h> 42 #include <sys/smp.h> 43 44 #include <vm/vm.h> 45 #include <vm/pmap.h> 46 47 #include <machine/bus.h> 48 #include <machine/intr_machdep.h> 49 #include <machine/md_var.h> 50 #include <machine/rtas.h> 51 52 #include <dev/ofw/ofw_bus.h> 53 #include <dev/ofw/ofw_bus_subr.h> 54 55 #ifdef POWERNV 56 #include <powerpc/powernv/opal.h> 57 #endif 58 59 #include "phyp-hvcall.h" 60 #include "pic_if.h" 61 62 #define XICP_PRIORITY 5 /* Random non-zero number */ 63 #define XICP_IPI 2 64 #define MAX_XICP_IRQS (1<<24) /* 24-bit XIRR field */ 65 66 static int xicp_probe(device_t); 67 static int xicp_attach(device_t); 68 static int xics_probe(device_t); 69 static int xics_attach(device_t); 70 71 static void xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv); 72 static void xicp_dispatch(device_t, struct trapframe *); 73 static void xicp_enable(device_t, u_int, u_int, void **priv); 74 static void xicp_eoi(device_t, u_int, void *priv); 75 static void xicp_ipi(device_t, u_int); 76 static void xicp_mask(device_t, u_int, void *priv); 77 static void xicp_unmask(device_t, u_int, void *priv); 78 79 #ifdef POWERNV 80 extern void (*powernv_smp_ap_extra_init)(void); 81 static void xicp_smp_cpu_startup(void); 82 #endif 83 84 static device_method_t xicp_methods[] = { 85 /* Device interface */ 86 DEVMETHOD(device_probe, xicp_probe), 87 DEVMETHOD(device_attach, xicp_attach), 88 89 /* PIC interface */ 90 DEVMETHOD(pic_bind, xicp_bind), 91 DEVMETHOD(pic_dispatch, xicp_dispatch), 92 DEVMETHOD(pic_enable, xicp_enable), 93 DEVMETHOD(pic_eoi, xicp_eoi), 94 DEVMETHOD(pic_ipi, xicp_ipi), 95 DEVMETHOD(pic_mask, xicp_mask), 96 DEVMETHOD(pic_unmask, xicp_unmask), 97 98 DEVMETHOD_END 99 }; 100 101 static device_method_t xics_methods[] = { 102 /* Device interface */ 103 DEVMETHOD(device_probe, xics_probe), 104 DEVMETHOD(device_attach, xics_attach), 105 106 DEVMETHOD_END 107 }; 108 109 struct xicp_intvec { 110 int irq; 111 int vector; 112 int cpu; 113 }; 114 115 struct xicp_softc { 116 struct mtx sc_mtx; 117 struct resource *mem[MAXCPU]; 118 119 int cpu_range[2]; 120 121 int ibm_int_on; 122 int ibm_int_off; 123 int ibm_get_xive; 124 int ibm_set_xive; 125 126 /* XXX: inefficient -- hash table? tree? */ 127 struct xicp_intvec intvecs[256]; 128 int nintvecs; 129 int ipi_vec; 130 bool xics_emu; 131 }; 132 133 static driver_t xicp_driver = { 134 "xicp", 135 xicp_methods, 136 sizeof(struct xicp_softc) 137 }; 138 139 static driver_t xics_driver = { 140 "xics", 141 xics_methods, 142 0 143 }; 144 145 #ifdef POWERNV 146 /* We can only pass physical addresses into OPAL. Kernel stacks are in the KVA, 147 * not in the direct map, so we need to somehow extract the physical address. 148 * However, pmap_kextract() takes locks, which is forbidden in a critical region 149 * (which PIC_DISPATCH() operates in). The kernel is mapped into the Direct 150 * Map (0xc000....), and the CPU implicitly drops the top two bits when doing 151 * real address by nature that the bus width is smaller than 64-bits. Placing 152 * cpu_xirr into the DMAP lets us take advantage of this and avoids the 153 * pmap_kextract() that would otherwise be needed if using the stack variable. 154 */ 155 static uint32_t cpu_xirr[MAXCPU]; 156 #endif 157 158 static devclass_t xicp_devclass; 159 static devclass_t xics_devclass; 160 161 EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, xicp_devclass, 0, 0, 162 BUS_PASS_INTERRUPT-1); 163 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, xics_devclass, 0, 0, 164 BUS_PASS_INTERRUPT); 165 166 #ifdef POWERNV 167 static struct resource * 168 xicp_mem_for_cpu(int cpu) 169 { 170 devclass_t dc; 171 device_t dev; 172 struct xicp_softc *sc; 173 int i; 174 175 dc = devclass_find(xicp_driver.name); 176 for (i = 0; (dev = devclass_get_device(dc, i)) != NULL; i++){ 177 sc = device_get_softc(dev); 178 if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1]) 179 return (sc->mem[cpu - sc->cpu_range[0]]); 180 } 181 182 return (NULL); 183 } 184 #endif 185 186 static int 187 xicp_probe(device_t dev) 188 { 189 190 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") && 191 !ofw_bus_is_compatible(dev, "ibm,opal-intc")) 192 return (ENXIO); 193 194 device_set_desc(dev, "External Interrupt Presentation Controller"); 195 return (BUS_PROBE_GENERIC); 196 } 197 198 static int 199 xics_probe(device_t dev) 200 { 201 202 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") && 203 !ofw_bus_is_compatible(dev, "IBM,opal-xics")) 204 return (ENXIO); 205 206 device_set_desc(dev, "External Interrupt Source Controller"); 207 return (BUS_PROBE_GENERIC); 208 } 209 210 static int 211 xicp_attach(device_t dev) 212 { 213 struct xicp_softc *sc = device_get_softc(dev); 214 phandle_t phandle = ofw_bus_get_node(dev); 215 216 if (rtas_exists()) { 217 sc->ibm_int_on = rtas_token_lookup("ibm,int-on"); 218 sc->ibm_int_off = rtas_token_lookup("ibm,int-off"); 219 sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive"); 220 sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive"); 221 #ifdef POWERNV 222 } else if (opal_check() == 0) { 223 /* No init needed */ 224 #endif 225 } else { 226 device_printf(dev, "Cannot attach without RTAS or OPAL\n"); 227 return (ENXIO); 228 } 229 230 if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) { 231 OF_getencprop(phandle, "ibm,interrupt-server-ranges", 232 sc->cpu_range, sizeof(sc->cpu_range)); 233 sc->cpu_range[1] += sc->cpu_range[0]; 234 device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0], 235 sc->cpu_range[1]-1); 236 #ifdef POWERNV 237 } else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) { 238 /* 239 * For now run POWER9 XIVE interrupt controller in XICS 240 * compatibility mode. 241 */ 242 sc->xics_emu = true; 243 opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EMU); 244 #endif 245 } else { 246 sc->cpu_range[0] = 0; 247 sc->cpu_range[1] = mp_ncpus; 248 } 249 250 #ifdef POWERNV 251 if (mfmsr() & PSL_HV) { 252 int i; 253 254 if (sc->xics_emu) { 255 opal_call(OPAL_INT_SET_CPPR, 0xff); 256 for (i = 0; i < mp_ncpus; i++) { 257 opal_call(OPAL_INT_SET_MFRR, 258 pcpu_find(i)->pc_hwref, 0xff); 259 } 260 } else { 261 for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) { 262 sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 263 &i, RF_ACTIVE); 264 if (sc->mem[i] == NULL) { 265 device_printf(dev, "Could not alloc mem " 266 "resource %d\n", i); 267 return (ENXIO); 268 } 269 270 /* Unmask interrupts on all cores */ 271 bus_write_1(sc->mem[i], 4, 0xff); 272 bus_write_1(sc->mem[i], 12, 0xff); 273 } 274 } 275 } 276 #endif 277 278 mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF); 279 sc->nintvecs = 0; 280 281 powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS, 282 1 /* Number of IPIs */, FALSE); 283 root_pic = dev; 284 285 #ifdef POWERNV 286 if (sc->xics_emu) 287 powernv_smp_ap_extra_init = xicp_smp_cpu_startup; 288 #endif 289 290 return (0); 291 } 292 293 static int 294 xics_attach(device_t dev) 295 { 296 phandle_t phandle = ofw_bus_get_node(dev); 297 298 /* The XICP (root PIC) will handle all our interrupts */ 299 powerpc_register_pic(root_pic, OF_xref_from_node(phandle), 300 MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE); 301 302 return (0); 303 } 304 305 static __inline struct xicp_intvec * 306 xicp_setup_priv(struct xicp_softc *sc, u_int irq, void **priv) 307 { 308 if (*priv == NULL) { 309 KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs), 310 ("Too many XICP interrupts")); 311 mtx_lock(&sc->sc_mtx); 312 *priv = &sc->intvecs[sc->nintvecs++]; 313 mtx_unlock(&sc->sc_mtx); 314 } 315 316 return (*priv); 317 } 318 319 /* 320 * PIC I/F methods. 321 */ 322 323 static void 324 xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv) 325 { 326 struct xicp_softc *sc = device_get_softc(dev); 327 struct xicp_intvec *iv; 328 cell_t status, cpu; 329 int ncpus, i, error = -1; 330 331 /* Ignore IPIs */ 332 if (irq == MAX_XICP_IRQS) 333 return; 334 335 iv = xicp_setup_priv(sc, irq, priv); 336 337 /* 338 * This doesn't appear to actually support affinity groups, so pick a 339 * random CPU. 340 */ 341 ncpus = 0; 342 CPU_FOREACH(cpu) 343 if (CPU_ISSET(cpu, &cpumask)) ncpus++; 344 345 i = mftb() % ncpus; 346 ncpus = 0; 347 CPU_FOREACH(cpu) { 348 if (!CPU_ISSET(cpu, &cpumask)) 349 continue; 350 if (ncpus == i) 351 break; 352 ncpus++; 353 } 354 355 cpu = pcpu_find(cpu)->pc_hwref; 356 iv->cpu = cpu; 357 358 if (rtas_exists()) 359 error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu, 360 XICP_PRIORITY, &status); 361 #ifdef POWERNV 362 else 363 error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY); 364 #endif 365 366 if (error < 0) 367 panic("Cannot bind interrupt %d to CPU %d", irq, cpu); 368 } 369 370 static void 371 xicp_dispatch(device_t dev, struct trapframe *tf) 372 { 373 struct xicp_softc *sc; 374 struct resource *regs = NULL; 375 uint64_t xirr, junk; 376 int i; 377 378 sc = device_get_softc(dev); 379 #ifdef POWERNV 380 if ((mfmsr() & PSL_HV) && !sc->xics_emu) { 381 regs = xicp_mem_for_cpu(PCPU_GET(hwref)); 382 KASSERT(regs != NULL, 383 ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref))); 384 } 385 #endif 386 387 for (;;) { 388 /* Return value in R4, use the PFT call */ 389 if (regs) { 390 xirr = bus_read_4(regs, 4); 391 #ifdef POWERNV 392 } else if (sc->xics_emu) { 393 opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)], 394 false); 395 xirr = cpu_xirr[PCPU_GET(cpuid)]; 396 #endif 397 } else { 398 /* Return value in R4, use the PFT call */ 399 phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk); 400 } 401 xirr &= 0x00ffffff; 402 403 if (xirr == 0) /* No more pending interrupts? */ 404 break; 405 406 if (xirr == XICP_IPI) { /* Magic number for IPIs */ 407 xirr = MAX_XICP_IRQS; /* Map to FreeBSD magic */ 408 409 /* Clear IPI */ 410 if (regs) 411 bus_write_1(regs, 12, 0xff); 412 #ifdef POWERNV 413 else if (sc->xics_emu) 414 opal_call(OPAL_INT_SET_MFRR, 415 PCPU_GET(hwref), 0xff); 416 #endif 417 else 418 phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)), 419 0xff); 420 i = sc->ipi_vec; 421 } else { 422 /* XXX: super inefficient */ 423 for (i = 0; i < sc->nintvecs; i++) { 424 if (sc->intvecs[i].irq == xirr) 425 break; 426 } 427 KASSERT(i < sc->nintvecs, ("Unmapped XIRR")); 428 } 429 430 powerpc_dispatch_intr(sc->intvecs[i].vector, tf); 431 } 432 } 433 434 static void 435 xicp_enable(device_t dev, u_int irq, u_int vector, void **priv) 436 { 437 struct xicp_softc *sc; 438 struct xicp_intvec *intr; 439 cell_t status, cpu; 440 441 sc = device_get_softc(dev); 442 443 /* Bind to this CPU to start: distrib. ID is last entry in gserver# */ 444 cpu = PCPU_GET(hwref); 445 446 intr = xicp_setup_priv(sc, irq, priv); 447 448 intr->irq = irq; 449 intr->vector = vector; 450 intr->cpu = cpu; 451 mb(); 452 453 /* IPIs are also enabled. Stash off the vector index */ 454 if (irq == MAX_XICP_IRQS) { 455 sc->ipi_vec = intr - sc->intvecs; 456 return; 457 } 458 459 if (rtas_exists()) { 460 rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu, 461 XICP_PRIORITY, &status); 462 xicp_unmask(dev, irq, intr); 463 #ifdef POWERNV 464 } else { 465 status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY); 466 /* Unmask implicit for OPAL */ 467 468 if (status != 0) 469 panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq, 470 cpu, status); 471 #endif 472 } 473 } 474 475 static void 476 xicp_eoi(device_t dev, u_int irq, void *priv) 477 { 478 #ifdef POWERNV 479 struct xicp_softc *sc; 480 #endif 481 uint64_t xirr; 482 483 if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */ 484 irq = XICP_IPI; 485 xirr = irq | (0xff << 24); 486 487 #ifdef POWERNV 488 if (mfmsr() & PSL_HV) { 489 sc = device_get_softc(dev); 490 if (sc->xics_emu) 491 opal_call(OPAL_INT_EOI, xirr); 492 else 493 bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr); 494 } else 495 #endif 496 phyp_hcall(H_EOI, xirr); 497 } 498 499 static void 500 xicp_ipi(device_t dev, u_int cpu) 501 { 502 503 #ifdef POWERNV 504 struct xicp_softc *sc; 505 cpu = pcpu_find(cpu)->pc_hwref; 506 507 if (mfmsr() & PSL_HV) { 508 sc = device_get_softc(dev); 509 if (sc->xics_emu) { 510 int64_t rv; 511 rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY); 512 if (rv != 0) 513 device_printf(dev, "IPI SET_MFRR result: %ld\n", rv); 514 } else 515 bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY); 516 } else 517 #endif 518 phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY); 519 } 520 521 static void 522 xicp_mask(device_t dev, u_int irq, void *priv) 523 { 524 struct xicp_softc *sc = device_get_softc(dev); 525 cell_t status; 526 527 if (irq == MAX_XICP_IRQS) 528 return; 529 530 if (rtas_exists()) { 531 rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status); 532 #ifdef POWERNV 533 } else { 534 struct xicp_intvec *ivec = priv; 535 536 KASSERT(ivec != NULL, ("Masking unconfigured interrupt")); 537 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, 0xff); 538 #endif 539 } 540 } 541 542 static void 543 xicp_unmask(device_t dev, u_int irq, void *priv) 544 { 545 struct xicp_softc *sc = device_get_softc(dev); 546 cell_t status; 547 548 if (irq == MAX_XICP_IRQS) 549 return; 550 551 if (rtas_exists()) { 552 rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status); 553 #ifdef POWERNV 554 } else { 555 struct xicp_intvec *ivec = priv; 556 557 KASSERT(ivec != NULL, ("Unmasking unconfigured interrupt")); 558 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, XICP_PRIORITY); 559 #endif 560 } 561 } 562 563 #ifdef POWERNV 564 /* This is only used on POWER9 systems with the XIVE's XICS emulation. */ 565 static void 566 xicp_smp_cpu_startup(void) 567 { 568 struct xicp_softc *sc; 569 570 if (mfmsr() & PSL_HV) { 571 sc = device_get_softc(root_pic); 572 573 if (sc->xics_emu) 574 opal_call(OPAL_INT_SET_CPPR, 0xff); 575 } 576 } 577 #endif 578