1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright 2011 Nathan Whitehorn 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_platform.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/module.h> 36 #include <sys/bus.h> 37 #include <sys/conf.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/smp.h> 41 42 #include <vm/vm.h> 43 #include <vm/pmap.h> 44 45 #include <machine/bus.h> 46 #include <machine/intr_machdep.h> 47 #include <machine/md_var.h> 48 #include <machine/rtas.h> 49 50 #include <dev/ofw/ofw_bus.h> 51 #include <dev/ofw/ofw_bus_subr.h> 52 53 #ifdef POWERNV 54 #include <powerpc/powernv/opal.h> 55 #endif 56 57 #include "phyp-hvcall.h" 58 #include "pic_if.h" 59 60 #define XICP_PRIORITY 5 /* Random non-zero number */ 61 #define XICP_IPI 2 62 #define MAX_XICP_IRQS (1<<24) /* 24-bit XIRR field */ 63 64 #define XIVE_XICS_MODE_EMU 0 65 #define XIVE_XICS_MODE_EXP 1 66 67 static int xicp_probe(device_t); 68 static int xicp_attach(device_t); 69 static int xics_probe(device_t); 70 static int xics_attach(device_t); 71 72 static void xicp_bind(device_t dev, u_int irq, cpuset_t cpumask); 73 static void xicp_dispatch(device_t, struct trapframe *); 74 static void xicp_enable(device_t, u_int, u_int); 75 static void xicp_eoi(device_t, u_int); 76 static void xicp_ipi(device_t, u_int); 77 static void xicp_mask(device_t, u_int); 78 static void xicp_unmask(device_t, u_int); 79 80 #ifdef POWERNV 81 void xicp_smp_cpu_startup(void); 82 #endif 83 84 static device_method_t xicp_methods[] = { 85 /* Device interface */ 86 DEVMETHOD(device_probe, xicp_probe), 87 DEVMETHOD(device_attach, xicp_attach), 88 89 /* PIC interface */ 90 DEVMETHOD(pic_bind, xicp_bind), 91 DEVMETHOD(pic_dispatch, xicp_dispatch), 92 DEVMETHOD(pic_enable, xicp_enable), 93 DEVMETHOD(pic_eoi, xicp_eoi), 94 DEVMETHOD(pic_ipi, xicp_ipi), 95 DEVMETHOD(pic_mask, xicp_mask), 96 DEVMETHOD(pic_unmask, xicp_unmask), 97 98 DEVMETHOD_END 99 }; 100 101 static device_method_t xics_methods[] = { 102 /* Device interface */ 103 DEVMETHOD(device_probe, xics_probe), 104 DEVMETHOD(device_attach, xics_attach), 105 106 DEVMETHOD_END 107 }; 108 109 struct xicp_softc { 110 struct mtx sc_mtx; 111 struct resource *mem[MAXCPU]; 112 113 int cpu_range[2]; 114 115 int ibm_int_on; 116 int ibm_int_off; 117 int ibm_get_xive; 118 int ibm_set_xive; 119 120 /* XXX: inefficient -- hash table? tree? */ 121 struct { 122 int irq; 123 int vector; 124 int cpu; 125 } intvecs[256]; 126 int nintvecs; 127 bool xics_emu; 128 }; 129 130 static driver_t xicp_driver = { 131 "xicp", 132 xicp_methods, 133 sizeof(struct xicp_softc) 134 }; 135 136 static driver_t xics_driver = { 137 "xics", 138 xics_methods, 139 0 140 }; 141 142 static uint32_t cpu_xirr[MAXCPU]; 143 144 static devclass_t xicp_devclass; 145 static devclass_t xics_devclass; 146 147 EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, xicp_devclass, 0, 0, 148 BUS_PASS_INTERRUPT-1); 149 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, xics_devclass, 0, 0, 150 BUS_PASS_INTERRUPT); 151 152 #ifdef POWERNV 153 static struct resource * 154 xicp_mem_for_cpu(int cpu) 155 { 156 device_t dev; 157 struct xicp_softc *sc; 158 int i; 159 160 for (i = 0; (dev = devclass_get_device(xicp_devclass, i)) != NULL; i++){ 161 sc = device_get_softc(dev); 162 if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1]) 163 return (sc->mem[cpu - sc->cpu_range[0]]); 164 } 165 166 return (NULL); 167 } 168 #endif 169 170 static int 171 xicp_probe(device_t dev) 172 { 173 174 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") && 175 !ofw_bus_is_compatible(dev, "ibm,opal-intc")) 176 return (ENXIO); 177 178 device_set_desc(dev, "External Interrupt Presentation Controller"); 179 return (BUS_PROBE_GENERIC); 180 } 181 182 static int 183 xics_probe(device_t dev) 184 { 185 186 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") && 187 !ofw_bus_is_compatible(dev, "IBM,opal-xics")) 188 return (ENXIO); 189 190 device_set_desc(dev, "External Interrupt Source Controller"); 191 return (BUS_PROBE_GENERIC); 192 } 193 194 static int 195 xicp_attach(device_t dev) 196 { 197 struct xicp_softc *sc = device_get_softc(dev); 198 phandle_t phandle = ofw_bus_get_node(dev); 199 200 if (rtas_exists()) { 201 sc->ibm_int_on = rtas_token_lookup("ibm,int-on"); 202 sc->ibm_int_off = rtas_token_lookup("ibm,int-off"); 203 sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive"); 204 sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive"); 205 #ifdef POWERNV 206 } else if (opal_check() == 0) { 207 /* No init needed */ 208 #endif 209 } else { 210 device_printf(dev, "Cannot attach without RTAS or OPAL\n"); 211 return (ENXIO); 212 } 213 214 if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) { 215 OF_getencprop(phandle, "ibm,interrupt-server-ranges", 216 sc->cpu_range, sizeof(sc->cpu_range)); 217 sc->cpu_range[1] += sc->cpu_range[0]; 218 device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0], 219 sc->cpu_range[1]-1); 220 #ifdef POWERNV 221 } else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) { 222 /* 223 * For now run POWER9 XIVE interrupt controller in XICS 224 * compatibility mode. 225 */ 226 sc->xics_emu = true; 227 opal_call(OPAL_XIVE_RESET, XIVE_XICS_MODE_EMU); 228 #endif 229 } else { 230 sc->cpu_range[0] = 0; 231 sc->cpu_range[1] = mp_ncpus; 232 } 233 234 #ifdef POWERNV 235 if (mfmsr() & PSL_HV) { 236 int i; 237 238 if (sc->xics_emu) { 239 opal_call(OPAL_INT_SET_CPPR, 0xff); 240 for (i = 0; i < mp_ncpus; i++) { 241 opal_call(OPAL_INT_SET_MFRR, 242 pcpu_find(i)->pc_hwref, 0xff); 243 } 244 } else { 245 for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) { 246 sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 247 &i, RF_ACTIVE); 248 if (sc->mem[i] == NULL) { 249 device_printf(dev, "Could not alloc mem " 250 "resource %d\n", i); 251 return (ENXIO); 252 } 253 254 /* Unmask interrupts on all cores */ 255 bus_write_1(sc->mem[i], 4, 0xff); 256 bus_write_1(sc->mem[i], 12, 0xff); 257 } 258 } 259 } 260 #endif 261 262 mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF); 263 sc->nintvecs = 0; 264 265 powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS, 266 1 /* Number of IPIs */, FALSE); 267 root_pic = dev; 268 269 return (0); 270 } 271 272 static int 273 xics_attach(device_t dev) 274 { 275 phandle_t phandle = ofw_bus_get_node(dev); 276 277 /* The XICP (root PIC) will handle all our interrupts */ 278 powerpc_register_pic(root_pic, OF_xref_from_node(phandle), 279 MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE); 280 281 return (0); 282 } 283 284 /* 285 * PIC I/F methods. 286 */ 287 288 static void 289 xicp_bind(device_t dev, u_int irq, cpuset_t cpumask) 290 { 291 struct xicp_softc *sc = device_get_softc(dev); 292 cell_t status, cpu; 293 int ncpus, i, error; 294 295 /* Ignore IPIs */ 296 if (irq == MAX_XICP_IRQS) 297 return; 298 299 /* 300 * This doesn't appear to actually support affinity groups, so pick a 301 * random CPU. 302 */ 303 ncpus = 0; 304 CPU_FOREACH(cpu) 305 if (CPU_ISSET(cpu, &cpumask)) ncpus++; 306 307 i = mftb() % ncpus; 308 ncpus = 0; 309 CPU_FOREACH(cpu) { 310 if (!CPU_ISSET(cpu, &cpumask)) 311 continue; 312 if (ncpus == i) 313 break; 314 ncpus++; 315 } 316 317 cpu = pcpu_find(cpu)->pc_hwref; 318 319 /* XXX: super inefficient */ 320 for (i = 0; i < sc->nintvecs; i++) { 321 if (sc->intvecs[i].irq == irq) { 322 sc->intvecs[i].cpu = cpu; 323 break; 324 } 325 } 326 KASSERT(i < sc->nintvecs, ("Binding non-configured interrupt")); 327 328 if (rtas_exists()) 329 error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu, 330 XICP_PRIORITY, &status); 331 #ifdef POWERNV 332 else 333 error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY); 334 #endif 335 336 if (error < 0) 337 panic("Cannot bind interrupt %d to CPU %d", irq, cpu); 338 } 339 340 static void 341 xicp_dispatch(device_t dev, struct trapframe *tf) 342 { 343 struct xicp_softc *sc; 344 struct resource *regs = NULL; 345 uint64_t xirr, junk; 346 int i; 347 348 sc = device_get_softc(dev); 349 #ifdef POWERNV 350 if ((mfmsr() & PSL_HV) && !sc->xics_emu) { 351 regs = xicp_mem_for_cpu(PCPU_GET(hwref)); 352 KASSERT(regs != NULL, 353 ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref))); 354 } 355 #endif 356 357 for (;;) { 358 /* Return value in R4, use the PFT call */ 359 if (regs) { 360 xirr = bus_read_4(regs, 4); 361 #ifdef POWERNV 362 } else if (sc->xics_emu) { 363 opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)], 364 false); 365 xirr = cpu_xirr[PCPU_GET(cpuid)]; 366 #endif 367 } else { 368 /* Return value in R4, use the PFT call */ 369 phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk); 370 } 371 xirr &= 0x00ffffff; 372 373 if (xirr == 0) { /* No more pending interrupts? */ 374 if (regs) 375 bus_write_1(regs, 4, 0xff); 376 #ifdef POWERNV 377 else if (sc->xics_emu) 378 opal_call(OPAL_INT_SET_CPPR, 0xff); 379 #endif 380 else 381 phyp_hcall(H_CPPR, (uint64_t)0xff); 382 break; 383 } 384 if (xirr == XICP_IPI) { /* Magic number for IPIs */ 385 xirr = MAX_XICP_IRQS; /* Map to FreeBSD magic */ 386 387 /* Clear IPI */ 388 if (regs) 389 bus_write_1(regs, 12, 0xff); 390 #ifdef POWERNV 391 else if (sc->xics_emu) 392 opal_call(OPAL_INT_SET_MFRR, 393 PCPU_GET(hwref), 0xff); 394 #endif 395 else 396 phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)), 397 0xff); 398 } 399 400 /* XXX: super inefficient */ 401 for (i = 0; i < sc->nintvecs; i++) { 402 if (sc->intvecs[i].irq == xirr) 403 break; 404 } 405 406 KASSERT(i < sc->nintvecs, ("Unmapped XIRR")); 407 powerpc_dispatch_intr(sc->intvecs[i].vector, tf); 408 } 409 } 410 411 static void 412 xicp_enable(device_t dev, u_int irq, u_int vector) 413 { 414 struct xicp_softc *sc; 415 cell_t status, cpu; 416 417 sc = device_get_softc(dev); 418 419 KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs), 420 ("Too many XICP interrupts")); 421 422 /* Bind to this CPU to start: distrib. ID is last entry in gserver# */ 423 cpu = PCPU_GET(hwref); 424 425 mtx_lock(&sc->sc_mtx); 426 sc->intvecs[sc->nintvecs].irq = irq; 427 sc->intvecs[sc->nintvecs].vector = vector; 428 sc->intvecs[sc->nintvecs].cpu = cpu; 429 mb(); 430 sc->nintvecs++; 431 mtx_unlock(&sc->sc_mtx); 432 433 /* IPIs are also enabled */ 434 if (irq == MAX_XICP_IRQS) 435 return; 436 437 if (rtas_exists()) { 438 rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu, 439 XICP_PRIORITY, &status); 440 xicp_unmask(dev, irq); 441 #ifdef POWERNV 442 } else { 443 status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY); 444 /* Unmask implicit for OPAL */ 445 446 if (status != 0) 447 panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq, 448 cpu, status); 449 #endif 450 } 451 } 452 453 static void 454 xicp_eoi(device_t dev, u_int irq) 455 { 456 #ifdef POWERNV 457 struct xicp_softc *sc; 458 #endif 459 uint64_t xirr; 460 461 if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */ 462 irq = XICP_IPI; 463 xirr = irq | (XICP_PRIORITY << 24); 464 465 #ifdef POWERNV 466 if (mfmsr() & PSL_HV) { 467 sc = device_get_softc(dev); 468 if (sc->xics_emu) 469 opal_call(OPAL_INT_EOI, xirr); 470 else 471 bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr); 472 } else 473 #endif 474 phyp_hcall(H_EOI, xirr); 475 } 476 477 static void 478 xicp_ipi(device_t dev, u_int cpu) 479 { 480 481 #ifdef POWERNV 482 struct xicp_softc *sc; 483 cpu = pcpu_find(cpu)->pc_hwref; 484 485 if (mfmsr() & PSL_HV) { 486 sc = device_get_softc(dev); 487 if (sc->xics_emu) { 488 int64_t rv; 489 rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY); 490 if (rv != 0) 491 device_printf(dev, "IPI SET_MFRR result: %ld\n", rv); 492 } else 493 bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY); 494 } else 495 #endif 496 phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY); 497 } 498 499 static void 500 xicp_mask(device_t dev, u_int irq) 501 { 502 struct xicp_softc *sc = device_get_softc(dev); 503 cell_t status; 504 505 if (irq == MAX_XICP_IRQS) 506 return; 507 508 if (rtas_exists()) { 509 rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status); 510 #ifdef POWERNV 511 } else { 512 int i; 513 514 for (i = 0; i < sc->nintvecs; i++) { 515 if (sc->intvecs[i].irq == irq) { 516 break; 517 } 518 } 519 KASSERT(i < sc->nintvecs, ("Masking unconfigured interrupt")); 520 opal_call(OPAL_SET_XIVE, irq, sc->intvecs[i].cpu << 2, 0xff); 521 #endif 522 } 523 } 524 525 static void 526 xicp_unmask(device_t dev, u_int irq) 527 { 528 struct xicp_softc *sc = device_get_softc(dev); 529 cell_t status; 530 531 if (irq == MAX_XICP_IRQS) 532 return; 533 534 if (rtas_exists()) { 535 rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status); 536 #ifdef POWERNV 537 } else { 538 int i; 539 540 for (i = 0; i < sc->nintvecs; i++) { 541 if (sc->intvecs[i].irq == irq) { 542 break; 543 } 544 } 545 KASSERT(i < sc->nintvecs, ("Unmasking unconfigured interrupt")); 546 opal_call(OPAL_SET_XIVE, irq, sc->intvecs[i].cpu << 2, 547 XICP_PRIORITY); 548 #endif 549 } 550 } 551 552 #ifdef POWERNV 553 /* This is only used on POWER9 systems with the XIVE's XICS emulation. */ 554 void 555 xicp_smp_cpu_startup(void) 556 { 557 struct xicp_softc *sc; 558 559 if (mfmsr() & PSL_HV) { 560 sc = device_get_softc(root_pic); 561 562 if (sc->xics_emu) 563 opal_call(OPAL_INT_SET_CPPR, 0xff); 564 } 565 } 566 #endif 567