1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright 2011 Nathan Whitehorn 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 #include "opt_platform.h" 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/module.h> 34 #include <sys/bus.h> 35 #include <sys/conf.h> 36 #include <sys/kernel.h> 37 #include <sys/lock.h> 38 #include <sys/malloc.h> 39 #include <sys/mutex.h> 40 #include <sys/smp.h> 41 42 #include <vm/vm.h> 43 #include <vm/pmap.h> 44 45 #include <machine/bus.h> 46 #include <machine/intr_machdep.h> 47 #include <machine/md_var.h> 48 #include <machine/rtas.h> 49 50 #include <dev/ofw/ofw_bus.h> 51 #include <dev/ofw/ofw_bus_subr.h> 52 53 #ifdef POWERNV 54 #include <powerpc/powernv/opal.h> 55 #endif 56 57 #include "phyp-hvcall.h" 58 #include "pic_if.h" 59 60 #define XICP_PRIORITY 5 /* Random non-zero number */ 61 #define XICP_IPI 2 62 #define MAX_XICP_IRQS (1<<24) /* 24-bit XIRR field */ 63 64 static int xicp_probe(device_t); 65 static int xicp_attach(device_t); 66 static int xics_probe(device_t); 67 static int xics_attach(device_t); 68 69 static void xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv); 70 static void xicp_dispatch(device_t, struct trapframe *); 71 static void xicp_enable(device_t, u_int, u_int, void **priv); 72 static void xicp_eoi(device_t, u_int, void *priv); 73 static void xicp_ipi(device_t, u_int); 74 static void xicp_mask(device_t, u_int, void *priv); 75 static void xicp_unmask(device_t, u_int, void *priv); 76 77 #ifdef POWERNV 78 static void xicp_ap_init(device_t); 79 #endif 80 81 static device_method_t xicp_methods[] = { 82 /* Device interface */ 83 DEVMETHOD(device_probe, xicp_probe), 84 DEVMETHOD(device_attach, xicp_attach), 85 86 /* PIC interface */ 87 DEVMETHOD(pic_bind, xicp_bind), 88 DEVMETHOD(pic_dispatch, xicp_dispatch), 89 DEVMETHOD(pic_enable, xicp_enable), 90 DEVMETHOD(pic_eoi, xicp_eoi), 91 DEVMETHOD(pic_ipi, xicp_ipi), 92 DEVMETHOD(pic_mask, xicp_mask), 93 DEVMETHOD(pic_unmask, xicp_unmask), 94 #ifdef POWERNV 95 DEVMETHOD(pic_ap_init, xicp_ap_init), 96 #endif 97 98 DEVMETHOD_END 99 }; 100 101 static device_method_t xics_methods[] = { 102 /* Device interface */ 103 DEVMETHOD(device_probe, xics_probe), 104 DEVMETHOD(device_attach, xics_attach), 105 106 DEVMETHOD_END 107 }; 108 109 struct xicp_intvec { 110 int irq; 111 int vector; 112 int cpu; 113 }; 114 115 struct xicp_softc { 116 struct mtx sc_mtx; 117 struct resource *mem[MAXCPU]; 118 119 int cpu_range[2]; 120 121 int ibm_int_on; 122 int ibm_int_off; 123 int ibm_get_xive; 124 int ibm_set_xive; 125 126 /* XXX: inefficient -- hash table? tree? */ 127 struct xicp_intvec intvecs[256]; 128 int nintvecs; 129 int ipi_vec; 130 bool xics_emu; 131 }; 132 133 static driver_t xicp_driver = { 134 "xicp", 135 xicp_methods, 136 sizeof(struct xicp_softc) 137 }; 138 139 static driver_t xics_driver = { 140 "xics", 141 xics_methods, 142 0 143 }; 144 145 #ifdef POWERNV 146 /* We can only pass physical addresses into OPAL. Kernel stacks are in the KVA, 147 * not in the direct map, so we need to somehow extract the physical address. 148 * However, pmap_kextract() takes locks, which is forbidden in a critical region 149 * (which PIC_DISPATCH() operates in). The kernel is mapped into the Direct 150 * Map (0xc000....), and the CPU implicitly drops the top two bits when doing 151 * real address by nature that the bus width is smaller than 64-bits. Placing 152 * cpu_xirr into the DMAP lets us take advantage of this and avoids the 153 * pmap_kextract() that would otherwise be needed if using the stack variable. 154 */ 155 static uint32_t cpu_xirr[MAXCPU]; 156 #endif 157 158 EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, 0, 0, BUS_PASS_INTERRUPT - 1); 159 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, 0, 0, BUS_PASS_INTERRUPT); 160 161 #ifdef POWERNV 162 static struct resource * 163 xicp_mem_for_cpu(int cpu) 164 { 165 devclass_t dc; 166 device_t dev; 167 struct xicp_softc *sc; 168 int i; 169 170 dc = devclass_find(xicp_driver.name); 171 for (i = 0; (dev = devclass_get_device(dc, i)) != NULL; i++){ 172 sc = device_get_softc(dev); 173 if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1]) 174 return (sc->mem[cpu - sc->cpu_range[0]]); 175 } 176 177 return (NULL); 178 } 179 #endif 180 181 static int 182 xicp_probe(device_t dev) 183 { 184 185 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") && 186 !ofw_bus_is_compatible(dev, "ibm,opal-intc")) 187 return (ENXIO); 188 189 device_set_desc(dev, "External Interrupt Presentation Controller"); 190 return (BUS_PROBE_GENERIC); 191 } 192 193 static int 194 xics_probe(device_t dev) 195 { 196 197 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") && 198 !ofw_bus_is_compatible(dev, "IBM,opal-xics")) 199 return (ENXIO); 200 201 device_set_desc(dev, "External Interrupt Source Controller"); 202 return (BUS_PROBE_GENERIC); 203 } 204 205 static int 206 xicp_attach(device_t dev) 207 { 208 struct xicp_softc *sc = device_get_softc(dev); 209 phandle_t phandle = ofw_bus_get_node(dev); 210 211 if (rtas_exists()) { 212 sc->ibm_int_on = rtas_token_lookup("ibm,int-on"); 213 sc->ibm_int_off = rtas_token_lookup("ibm,int-off"); 214 sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive"); 215 sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive"); 216 #ifdef POWERNV 217 } else if (opal_check() == 0) { 218 /* No init needed */ 219 #endif 220 } else { 221 device_printf(dev, "Cannot attach without RTAS or OPAL\n"); 222 return (ENXIO); 223 } 224 225 if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) { 226 OF_getencprop(phandle, "ibm,interrupt-server-ranges", 227 sc->cpu_range, sizeof(sc->cpu_range)); 228 sc->cpu_range[1] += sc->cpu_range[0]; 229 device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0], 230 sc->cpu_range[1]-1); 231 #ifdef POWERNV 232 } else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) { 233 /* 234 * For now run POWER9 XIVE interrupt controller in XICS 235 * compatibility mode. 236 */ 237 sc->xics_emu = true; 238 opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EMU); 239 #endif 240 } else { 241 sc->cpu_range[0] = 0; 242 sc->cpu_range[1] = mp_ncpus; 243 } 244 245 #ifdef POWERNV 246 if (mfmsr() & PSL_HV) { 247 int i; 248 249 if (sc->xics_emu) { 250 opal_call(OPAL_INT_SET_CPPR, 0xff); 251 for (i = 0; i < mp_ncpus; i++) { 252 opal_call(OPAL_INT_SET_MFRR, 253 pcpu_find(i)->pc_hwref, 0xff); 254 } 255 } else { 256 for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) { 257 sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 258 &i, RF_ACTIVE); 259 if (sc->mem[i] == NULL) { 260 device_printf(dev, "Could not alloc mem " 261 "resource %d\n", i); 262 return (ENXIO); 263 } 264 265 /* Unmask interrupts on all cores */ 266 bus_write_1(sc->mem[i], 4, 0xff); 267 bus_write_1(sc->mem[i], 12, 0xff); 268 } 269 } 270 } 271 #endif 272 273 mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF); 274 sc->nintvecs = 0; 275 276 powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS, 277 1 /* Number of IPIs */, FALSE); 278 root_pic = dev; 279 280 return (0); 281 } 282 283 static int 284 xics_attach(device_t dev) 285 { 286 phandle_t phandle = ofw_bus_get_node(dev); 287 288 /* The XICP (root PIC) will handle all our interrupts */ 289 powerpc_register_pic(root_pic, OF_xref_from_node(phandle), 290 MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE); 291 292 return (0); 293 } 294 295 static __inline struct xicp_intvec * 296 xicp_setup_priv(struct xicp_softc *sc, u_int irq, void **priv) 297 { 298 if (*priv == NULL) { 299 KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs), 300 ("Too many XICP interrupts")); 301 mtx_lock(&sc->sc_mtx); 302 *priv = &sc->intvecs[sc->nintvecs++]; 303 mtx_unlock(&sc->sc_mtx); 304 } 305 306 return (*priv); 307 } 308 309 /* 310 * PIC I/F methods. 311 */ 312 313 static void 314 xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv) 315 { 316 struct xicp_softc *sc = device_get_softc(dev); 317 struct xicp_intvec *iv; 318 cell_t status, cpu; 319 int ncpus, i, error = -1; 320 321 /* Ignore IPIs */ 322 if (irq == MAX_XICP_IRQS) 323 return; 324 325 iv = xicp_setup_priv(sc, irq, priv); 326 327 /* 328 * This doesn't appear to actually support affinity groups, so pick a 329 * random CPU. 330 */ 331 ncpus = 0; 332 CPU_FOREACH(cpu) 333 if (CPU_ISSET(cpu, &cpumask)) ncpus++; 334 335 i = mftb() % ncpus; 336 ncpus = 0; 337 CPU_FOREACH(cpu) { 338 if (!CPU_ISSET(cpu, &cpumask)) 339 continue; 340 if (ncpus == i) 341 break; 342 ncpus++; 343 } 344 345 cpu = pcpu_find(cpu)->pc_hwref; 346 iv->cpu = cpu; 347 348 if (rtas_exists()) 349 error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu, 350 XICP_PRIORITY, &status); 351 #ifdef POWERNV 352 else 353 error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY); 354 #endif 355 356 if (error < 0) 357 panic("Cannot bind interrupt %d to CPU %d", irq, cpu); 358 } 359 360 static void 361 xicp_dispatch(device_t dev, struct trapframe *tf) 362 { 363 struct xicp_softc *sc; 364 struct resource *regs = NULL; 365 uint64_t xirr, junk; 366 int i; 367 368 sc = device_get_softc(dev); 369 #ifdef POWERNV 370 if ((mfmsr() & PSL_HV) && !sc->xics_emu) { 371 regs = xicp_mem_for_cpu(PCPU_GET(hwref)); 372 KASSERT(regs != NULL, 373 ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref))); 374 } 375 #endif 376 377 for (;;) { 378 /* Return value in R4, use the PFT call */ 379 if (regs) { 380 xirr = bus_read_4(regs, 4); 381 #ifdef POWERNV 382 } else if (sc->xics_emu) { 383 opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)], 384 false); 385 xirr = cpu_xirr[PCPU_GET(cpuid)]; 386 #endif 387 } else { 388 /* Return value in R4, use the PFT call */ 389 phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk); 390 } 391 xirr &= 0x00ffffff; 392 393 if (xirr == 0) /* No more pending interrupts? */ 394 break; 395 396 if (xirr == XICP_IPI) { /* Magic number for IPIs */ 397 xirr = MAX_XICP_IRQS; /* Map to FreeBSD magic */ 398 399 /* Clear IPI */ 400 if (regs) 401 bus_write_1(regs, 12, 0xff); 402 #ifdef POWERNV 403 else if (sc->xics_emu) 404 opal_call(OPAL_INT_SET_MFRR, 405 PCPU_GET(hwref), 0xff); 406 #endif 407 else 408 phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)), 409 0xff); 410 i = sc->ipi_vec; 411 } else { 412 /* XXX: super inefficient */ 413 for (i = 0; i < sc->nintvecs; i++) { 414 if (sc->intvecs[i].irq == xirr) 415 break; 416 } 417 KASSERT(i < sc->nintvecs, ("Unmapped XIRR")); 418 } 419 420 powerpc_dispatch_intr(sc->intvecs[i].vector, tf); 421 } 422 } 423 424 static void 425 xicp_enable(device_t dev, u_int irq, u_int vector, void **priv) 426 { 427 struct xicp_softc *sc; 428 struct xicp_intvec *intr; 429 cell_t status, cpu; 430 431 sc = device_get_softc(dev); 432 433 /* Bind to this CPU to start: distrib. ID is last entry in gserver# */ 434 cpu = PCPU_GET(hwref); 435 436 intr = xicp_setup_priv(sc, irq, priv); 437 438 intr->irq = irq; 439 intr->vector = vector; 440 intr->cpu = cpu; 441 mb(); 442 443 /* IPIs are also enabled. Stash off the vector index */ 444 if (irq == MAX_XICP_IRQS) { 445 sc->ipi_vec = intr - sc->intvecs; 446 return; 447 } 448 449 if (rtas_exists()) { 450 rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu, 451 XICP_PRIORITY, &status); 452 xicp_unmask(dev, irq, intr); 453 #ifdef POWERNV 454 } else { 455 status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY); 456 /* Unmask implicit for OPAL */ 457 458 if (status != 0) 459 panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq, 460 cpu, status); 461 #endif 462 } 463 } 464 465 static void 466 xicp_eoi(device_t dev, u_int irq, void *priv) 467 { 468 #ifdef POWERNV 469 struct xicp_softc *sc; 470 #endif 471 uint64_t xirr; 472 473 if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */ 474 irq = XICP_IPI; 475 xirr = irq | (0xff << 24); 476 477 #ifdef POWERNV 478 if (mfmsr() & PSL_HV) { 479 sc = device_get_softc(dev); 480 if (sc->xics_emu) 481 opal_call(OPAL_INT_EOI, xirr); 482 else 483 bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr); 484 } else 485 #endif 486 phyp_hcall(H_EOI, xirr); 487 } 488 489 static void 490 xicp_ipi(device_t dev, u_int cpu) 491 { 492 493 #ifdef POWERNV 494 struct xicp_softc *sc; 495 cpu = pcpu_find(cpu)->pc_hwref; 496 497 if (mfmsr() & PSL_HV) { 498 sc = device_get_softc(dev); 499 if (sc->xics_emu) { 500 int64_t rv; 501 rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY); 502 if (rv != 0) 503 device_printf(dev, "IPI SET_MFRR result: %ld\n", rv); 504 } else 505 bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY); 506 } else 507 #endif 508 phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY); 509 } 510 511 static void 512 xicp_mask(device_t dev, u_int irq, void *priv) 513 { 514 struct xicp_softc *sc = device_get_softc(dev); 515 cell_t status; 516 517 if (irq == MAX_XICP_IRQS) 518 return; 519 520 if (rtas_exists()) { 521 rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status); 522 #ifdef POWERNV 523 } else { 524 struct xicp_intvec *ivec = priv; 525 526 KASSERT(ivec != NULL, ("Masking unconfigured interrupt")); 527 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, 0xff); 528 #endif 529 } 530 } 531 532 static void 533 xicp_unmask(device_t dev, u_int irq, void *priv) 534 { 535 struct xicp_softc *sc = device_get_softc(dev); 536 cell_t status; 537 538 if (irq == MAX_XICP_IRQS) 539 return; 540 541 if (rtas_exists()) { 542 rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status); 543 #ifdef POWERNV 544 } else { 545 struct xicp_intvec *ivec = priv; 546 547 KASSERT(ivec != NULL, ("Unmasking unconfigured interrupt")); 548 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, XICP_PRIORITY); 549 #endif 550 } 551 } 552 553 #ifdef POWERNV 554 /* Only meaningful on POWER9 systems using the XIVE's XICS-emulation mode. */ 555 static void 556 xicp_ap_init(device_t dev) 557 { 558 struct xicp_softc *sc; 559 560 if (mfmsr() & PSL_HV) { 561 sc = device_get_softc(dev); 562 563 if (sc->xics_emu) 564 opal_call(OPAL_INT_SET_CPPR, 0xff); 565 } 566 } 567 #endif 568