1d49fc192SJustin Hibbits /*- 2d49fc192SJustin Hibbits * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3d49fc192SJustin Hibbits * 4d49fc192SJustin Hibbits * Copyright 2019 Justin Hibbits 5d49fc192SJustin Hibbits * 6d49fc192SJustin Hibbits * Redistribution and use in source and binary forms, with or without 7d49fc192SJustin Hibbits * modification, are permitted provided that the following conditions 8d49fc192SJustin Hibbits * are met: 9d49fc192SJustin Hibbits * 1. Redistributions of source code must retain the above copyright 10d49fc192SJustin Hibbits * notice, this list of conditions and the following disclaimer. 11d49fc192SJustin Hibbits * 2. Redistributions in binary form must reproduce the above copyright 12d49fc192SJustin Hibbits * notice, this list of conditions and the following disclaimer in the 13d49fc192SJustin Hibbits * documentation and/or other materials provided with the distribution. 14d49fc192SJustin Hibbits * 15d49fc192SJustin Hibbits * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16d49fc192SJustin Hibbits * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17d49fc192SJustin Hibbits * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18d49fc192SJustin Hibbits * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19d49fc192SJustin Hibbits * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 20d49fc192SJustin Hibbits * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21d49fc192SJustin Hibbits * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 22d49fc192SJustin Hibbits * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23d49fc192SJustin Hibbits * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24d49fc192SJustin Hibbits * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25d49fc192SJustin Hibbits * SUCH DAMAGE. 26d49fc192SJustin Hibbits */ 27d49fc192SJustin Hibbits 28d49fc192SJustin Hibbits #include <sys/cdefs.h> 29d49fc192SJustin Hibbits __FBSDID("$FreeBSD$"); 30d49fc192SJustin Hibbits 31d49fc192SJustin Hibbits #include "opt_platform.h" 32d49fc192SJustin Hibbits 33d49fc192SJustin Hibbits #include <sys/param.h> 34d49fc192SJustin Hibbits #include <sys/systm.h> 35d49fc192SJustin Hibbits #include <sys/module.h> 36d49fc192SJustin Hibbits #include <sys/bus.h> 37d49fc192SJustin Hibbits #include <sys/conf.h> 38d49fc192SJustin Hibbits #include <sys/endian.h> 39d49fc192SJustin Hibbits #include <sys/kernel.h> 40e2e050c8SConrad Meyer #include <sys/lock.h> 41d49fc192SJustin Hibbits #include <sys/malloc.h> 42e2e050c8SConrad Meyer #include <sys/mutex.h> 43d49fc192SJustin Hibbits #include <sys/smp.h> 44d49fc192SJustin Hibbits 45d49fc192SJustin Hibbits #include <vm/vm.h> 46d49fc192SJustin Hibbits #include <vm/pmap.h> 47d49fc192SJustin Hibbits 48d49fc192SJustin Hibbits #include <machine/bus.h> 49d49fc192SJustin Hibbits #include <machine/intr_machdep.h> 50d49fc192SJustin Hibbits #include <machine/md_var.h> 51d49fc192SJustin Hibbits 52d49fc192SJustin Hibbits #include <dev/ofw/ofw_bus.h> 53d49fc192SJustin Hibbits #include <dev/ofw/ofw_bus_subr.h> 54d49fc192SJustin Hibbits 55d49fc192SJustin Hibbits #ifdef POWERNV 56d49fc192SJustin Hibbits #include <powerpc/powernv/opal.h> 57d49fc192SJustin Hibbits #endif 58d49fc192SJustin Hibbits 59d49fc192SJustin Hibbits #include "pic_if.h" 60d49fc192SJustin Hibbits 61d49fc192SJustin Hibbits #define XIVE_PRIORITY 7 /* Random non-zero number */ 62d49fc192SJustin Hibbits #define MAX_XIVE_IRQS (1<<24) /* 24-bit XIRR field */ 63d49fc192SJustin Hibbits 64d49fc192SJustin Hibbits /* Registers */ 65d49fc192SJustin Hibbits #define XIVE_TM_QW1_OS 0x010 /* Guest OS registers */ 66d49fc192SJustin Hibbits #define XIVE_TM_QW2_HV_POOL 0x020 /* Hypervisor pool registers */ 67d49fc192SJustin Hibbits #define XIVE_TM_QW3_HV 0x030 /* Hypervisor registers */ 68d49fc192SJustin Hibbits 69d49fc192SJustin Hibbits #define XIVE_TM_NSR 0x00 70d49fc192SJustin Hibbits #define XIVE_TM_CPPR 0x01 71d49fc192SJustin Hibbits #define XIVE_TM_IPB 0x02 72d49fc192SJustin Hibbits #define XIVE_TM_LSMFB 0x03 73d49fc192SJustin Hibbits #define XIVE_TM_ACK_CNT 0x04 74d49fc192SJustin Hibbits #define XIVE_TM_INC 0x05 75d49fc192SJustin Hibbits #define XIVE_TM_AGE 0x06 76d49fc192SJustin Hibbits #define XIVE_TM_PIPR 0x07 77d49fc192SJustin Hibbits 78d49fc192SJustin Hibbits #define TM_WORD0 0x0 79d49fc192SJustin Hibbits #define TM_WORD2 0x8 80d49fc192SJustin Hibbits #define TM_QW2W2_VP 0x80000000 81d49fc192SJustin Hibbits 82d49fc192SJustin Hibbits #define XIVE_TM_SPC_ACK 0x800 83d49fc192SJustin Hibbits #define TM_QW3NSR_HE_SHIFT 14 84d49fc192SJustin Hibbits #define TM_QW3_NSR_HE_NONE 0 85d49fc192SJustin Hibbits #define TM_QW3_NSR_HE_POOL 1 86d49fc192SJustin Hibbits #define TM_QW3_NSR_HE_PHYS 2 87d49fc192SJustin Hibbits #define TM_QW3_NSR_HE_LSI 3 88d49fc192SJustin Hibbits #define XIVE_TM_SPC_PULL_POOL_CTX 0x828 89d49fc192SJustin Hibbits 90d49fc192SJustin Hibbits #define XIVE_IRQ_LOAD_EOI 0x000 91d49fc192SJustin Hibbits #define XIVE_IRQ_STORE_EOI 0x400 92d49fc192SJustin Hibbits #define XIVE_IRQ_PQ_00 0xc00 93d49fc192SJustin Hibbits #define XIVE_IRQ_PQ_01 0xd00 94d49fc192SJustin Hibbits 95d49fc192SJustin Hibbits #define XIVE_IRQ_VAL_P 0x02 96d49fc192SJustin Hibbits #define XIVE_IRQ_VAL_Q 0x01 97d49fc192SJustin Hibbits 98d49fc192SJustin Hibbits struct xive_softc; 99d49fc192SJustin Hibbits struct xive_irq; 100d49fc192SJustin Hibbits 101d49fc192SJustin Hibbits extern void (*powernv_smp_ap_extra_init)(void); 102d49fc192SJustin Hibbits 103d49fc192SJustin Hibbits /* Private support */ 104d49fc192SJustin Hibbits static void xive_setup_cpu(void); 105d49fc192SJustin Hibbits static void xive_smp_cpu_startup(void); 106d49fc192SJustin Hibbits static void xive_init_irq(struct xive_irq *irqd, u_int irq); 107d49fc192SJustin Hibbits static struct xive_irq *xive_configure_irq(u_int irq); 108d49fc192SJustin Hibbits static int xive_provision_page(struct xive_softc *sc); 109d49fc192SJustin Hibbits 110d49fc192SJustin Hibbits /* Interfaces */ 111d49fc192SJustin Hibbits static int xive_probe(device_t); 112d49fc192SJustin Hibbits static int xive_attach(device_t); 113d49fc192SJustin Hibbits static int xics_probe(device_t); 114d49fc192SJustin Hibbits static int xics_attach(device_t); 115d49fc192SJustin Hibbits 116d49fc192SJustin Hibbits static void xive_bind(device_t, u_int, cpuset_t, void **); 117d49fc192SJustin Hibbits static void xive_dispatch(device_t, struct trapframe *); 118d49fc192SJustin Hibbits static void xive_enable(device_t, u_int, u_int, void **); 119d49fc192SJustin Hibbits static void xive_eoi(device_t, u_int, void *); 120d49fc192SJustin Hibbits static void xive_ipi(device_t, u_int); 121d49fc192SJustin Hibbits static void xive_mask(device_t, u_int, void *); 122d49fc192SJustin Hibbits static void xive_unmask(device_t, u_int, void *); 123d49fc192SJustin Hibbits static void xive_translate_code(device_t dev, u_int irq, int code, 124d49fc192SJustin Hibbits enum intr_trigger *trig, enum intr_polarity *pol); 125d49fc192SJustin Hibbits 126d49fc192SJustin Hibbits static device_method_t xive_methods[] = { 127d49fc192SJustin Hibbits /* Device interface */ 128d49fc192SJustin Hibbits DEVMETHOD(device_probe, xive_probe), 129d49fc192SJustin Hibbits DEVMETHOD(device_attach, xive_attach), 130d49fc192SJustin Hibbits 131d49fc192SJustin Hibbits /* PIC interface */ 132d49fc192SJustin Hibbits DEVMETHOD(pic_bind, xive_bind), 133d49fc192SJustin Hibbits DEVMETHOD(pic_dispatch, xive_dispatch), 134d49fc192SJustin Hibbits DEVMETHOD(pic_enable, xive_enable), 135d49fc192SJustin Hibbits DEVMETHOD(pic_eoi, xive_eoi), 136d49fc192SJustin Hibbits DEVMETHOD(pic_ipi, xive_ipi), 137d49fc192SJustin Hibbits DEVMETHOD(pic_mask, xive_mask), 138d49fc192SJustin Hibbits DEVMETHOD(pic_unmask, xive_unmask), 139d49fc192SJustin Hibbits DEVMETHOD(pic_translate_code, xive_translate_code), 140d49fc192SJustin Hibbits 141d49fc192SJustin Hibbits DEVMETHOD_END 142d49fc192SJustin Hibbits }; 143d49fc192SJustin Hibbits 144d49fc192SJustin Hibbits static device_method_t xics_methods[] = { 145d49fc192SJustin Hibbits /* Device interface */ 146d49fc192SJustin Hibbits DEVMETHOD(device_probe, xics_probe), 147d49fc192SJustin Hibbits DEVMETHOD(device_attach, xics_attach), 148d49fc192SJustin Hibbits 149d49fc192SJustin Hibbits DEVMETHOD_END 150d49fc192SJustin Hibbits }; 151d49fc192SJustin Hibbits 152d49fc192SJustin Hibbits struct xive_softc { 153d49fc192SJustin Hibbits struct mtx sc_mtx; 154d49fc192SJustin Hibbits struct resource *sc_mem; 155d49fc192SJustin Hibbits vm_size_t sc_prov_page_size; 156d49fc192SJustin Hibbits uint32_t sc_offset; 157d49fc192SJustin Hibbits }; 158d49fc192SJustin Hibbits 159d49fc192SJustin Hibbits struct xive_queue { 160d49fc192SJustin Hibbits uint32_t *q_page; 161d49fc192SJustin Hibbits uint32_t *q_eoi_page; 162d49fc192SJustin Hibbits uint32_t q_toggle; 163d49fc192SJustin Hibbits uint32_t q_size; 164d49fc192SJustin Hibbits uint32_t q_index; 165d49fc192SJustin Hibbits uint32_t q_mask; 166d49fc192SJustin Hibbits }; 167d49fc192SJustin Hibbits 168d49fc192SJustin Hibbits struct xive_irq { 169d49fc192SJustin Hibbits uint32_t girq; 170d49fc192SJustin Hibbits uint32_t lirq; 171d49fc192SJustin Hibbits uint64_t vp; 172d49fc192SJustin Hibbits uint64_t flags; 173d49fc192SJustin Hibbits #define OPAL_XIVE_IRQ_SHIFT_BUG 0x00000008 174d49fc192SJustin Hibbits #define OPAL_XIVE_IRQ_LSI 0x00000004 175d49fc192SJustin Hibbits #define OPAL_XIVE_IRQ_STORE_EOI 0x00000002 176d49fc192SJustin Hibbits #define OPAL_XIVE_IRQ_TRIGGER_PAGE 0x00000001 177d49fc192SJustin Hibbits uint8_t prio; 178d49fc192SJustin Hibbits vm_offset_t eoi_page; 179d49fc192SJustin Hibbits vm_offset_t trig_page; 180d49fc192SJustin Hibbits vm_size_t esb_size; 181d49fc192SJustin Hibbits int chip; 182d49fc192SJustin Hibbits }; 183d49fc192SJustin Hibbits 184d49fc192SJustin Hibbits struct xive_cpu { 185d49fc192SJustin Hibbits uint64_t vp; 186d49fc192SJustin Hibbits uint64_t flags; 187d49fc192SJustin Hibbits struct xive_irq ipi_data; 188d49fc192SJustin Hibbits struct xive_queue queue; /* We only use a single queue for now. */ 189d49fc192SJustin Hibbits uint64_t cam; 190d49fc192SJustin Hibbits uint32_t chip; 191d49fc192SJustin Hibbits }; 192d49fc192SJustin Hibbits 193d49fc192SJustin Hibbits static driver_t xive_driver = { 194d49fc192SJustin Hibbits "xive", 195d49fc192SJustin Hibbits xive_methods, 196d49fc192SJustin Hibbits sizeof(struct xive_softc) 197d49fc192SJustin Hibbits }; 198d49fc192SJustin Hibbits 199d49fc192SJustin Hibbits static driver_t xics_driver = { 200d49fc192SJustin Hibbits "xivevc", 201d49fc192SJustin Hibbits xics_methods, 202d49fc192SJustin Hibbits 0 203d49fc192SJustin Hibbits }; 204d49fc192SJustin Hibbits 205d49fc192SJustin Hibbits static devclass_t xive_devclass; 206d49fc192SJustin Hibbits static devclass_t xics_devclass; 207d49fc192SJustin Hibbits 208d49fc192SJustin Hibbits EARLY_DRIVER_MODULE(xive, ofwbus, xive_driver, xive_devclass, 0, 0, 209d49fc192SJustin Hibbits BUS_PASS_INTERRUPT-1); 210d49fc192SJustin Hibbits EARLY_DRIVER_MODULE(xivevc, ofwbus, xics_driver, xics_devclass, 0, 0, 211d49fc192SJustin Hibbits BUS_PASS_INTERRUPT); 212d49fc192SJustin Hibbits 213d49fc192SJustin Hibbits MALLOC_DEFINE(M_XIVE, "xive", "XIVE Memory"); 214d49fc192SJustin Hibbits 215d49fc192SJustin Hibbits DPCPU_DEFINE_STATIC(struct xive_cpu, xive_cpu_data); 216d49fc192SJustin Hibbits 217d49fc192SJustin Hibbits static int xive_ipi_vector = -1; 218d49fc192SJustin Hibbits 219d49fc192SJustin Hibbits /* 220d49fc192SJustin Hibbits * XIVE Exploitation mode driver. 221d49fc192SJustin Hibbits * 222d49fc192SJustin Hibbits * The XIVE, present in the POWER9 CPU, can run in two modes: XICS emulation 223d49fc192SJustin Hibbits * mode, and "Exploitation mode". XICS emulation mode is compatible with the 224d49fc192SJustin Hibbits * POWER8 and earlier XICS interrupt controller, using OPAL calls to emulate 225d49fc192SJustin Hibbits * hypervisor calls and memory accesses. Exploitation mode gives us raw access 226d49fc192SJustin Hibbits * to the XIVE MMIO, improving performance significantly. 227d49fc192SJustin Hibbits * 228d49fc192SJustin Hibbits * The XIVE controller is a very bizarre interrupt controller. It uses queues 229d49fc192SJustin Hibbits * in memory to pass interrupts around, and maps itself into 512GB of physical 230d49fc192SJustin Hibbits * device address space, giving each interrupt in the system one or more pages 231d49fc192SJustin Hibbits * of address space. An IRQ is tied to a virtual processor, which could be a 232d49fc192SJustin Hibbits * physical CPU thread, or a guest CPU thread (LPAR running on a physical 233d49fc192SJustin Hibbits * thread). Thus, the controller can route interrupts directly to guest OSes 234d49fc192SJustin Hibbits * bypassing processing by the hypervisor, thereby improving performance of the 235d49fc192SJustin Hibbits * guest OS. 236d49fc192SJustin Hibbits * 237d49fc192SJustin Hibbits * An IRQ, in addition to being tied to a virtual processor, has one or two 238d49fc192SJustin Hibbits * page mappings: an EOI page, and an optional trigger page. The trigger page 239d49fc192SJustin Hibbits * could be the same as the EOI page. Level-sensitive interrupts (LSIs) don't 240d49fc192SJustin Hibbits * have a trigger page, as they're external interrupts controlled by physical 241d49fc192SJustin Hibbits * lines. MSIs and IPIs have trigger pages. An IPI is really just another IRQ 242d49fc192SJustin Hibbits * in the XIVE, which is triggered by software. 243d49fc192SJustin Hibbits * 244d49fc192SJustin Hibbits * An interesting behavior of the XIVE controller is that oftentimes the 245d49fc192SJustin Hibbits * contents of an address location don't actually matter, but the direction of 246d49fc192SJustin Hibbits * the action is the signifier (read vs write), and the address is significant. 247d49fc192SJustin Hibbits * Hence, masking and unmasking an interrupt is done by reading different 248d49fc192SJustin Hibbits * addresses in the EOI page, and triggering an interrupt consists of writing to 249d49fc192SJustin Hibbits * the trigger page. 250d49fc192SJustin Hibbits * 251d49fc192SJustin Hibbits * Additionally, the MMIO region mapped is CPU-sensitive, just like the 252d49fc192SJustin Hibbits * per-processor register space (private access) in OpenPIC. In order for a CPU 253d49fc192SJustin Hibbits * to receive interrupts it must itself configure its CPPR (Current Processor 254d49fc192SJustin Hibbits * Priority Register), it cannot be set by any other processor. This 255d49fc192SJustin Hibbits * necessitates the xive_smp_cpu_startup() function. 256d49fc192SJustin Hibbits * 257d49fc192SJustin Hibbits * Queues are pages of memory, sized powers-of-two, that are shared with the 258d49fc192SJustin Hibbits * XIVE. The XIVE writes into the queue with an alternating polarity bit, which 259d49fc192SJustin Hibbits * flips when the queue wraps. 260d49fc192SJustin Hibbits */ 261d49fc192SJustin Hibbits 262d49fc192SJustin Hibbits /* 263d49fc192SJustin Hibbits * Offset-based read/write interfaces. 264d49fc192SJustin Hibbits */ 265d49fc192SJustin Hibbits static uint16_t 266d49fc192SJustin Hibbits xive_read_2(struct xive_softc *sc, bus_size_t offset) 267d49fc192SJustin Hibbits { 268d49fc192SJustin Hibbits 269d49fc192SJustin Hibbits return (bus_read_2(sc->sc_mem, sc->sc_offset + offset)); 270d49fc192SJustin Hibbits } 271d49fc192SJustin Hibbits 272d49fc192SJustin Hibbits static void 273d49fc192SJustin Hibbits xive_write_1(struct xive_softc *sc, bus_size_t offset, uint8_t val) 274d49fc192SJustin Hibbits { 275d49fc192SJustin Hibbits 276d49fc192SJustin Hibbits bus_write_1(sc->sc_mem, sc->sc_offset + offset, val); 277d49fc192SJustin Hibbits } 278d49fc192SJustin Hibbits 279d49fc192SJustin Hibbits /* EOI and Trigger page access interfaces. */ 280d49fc192SJustin Hibbits static uint64_t 281d49fc192SJustin Hibbits xive_read_mmap8(vm_offset_t addr) 282d49fc192SJustin Hibbits { 283d49fc192SJustin Hibbits return (*(volatile uint64_t *)addr); 284d49fc192SJustin Hibbits } 285d49fc192SJustin Hibbits 286d49fc192SJustin Hibbits static void 287d49fc192SJustin Hibbits xive_write_mmap8(vm_offset_t addr, uint64_t val) 288d49fc192SJustin Hibbits { 289d49fc192SJustin Hibbits *(uint64_t *)(addr) = val; 290d49fc192SJustin Hibbits } 291d49fc192SJustin Hibbits 292d49fc192SJustin Hibbits /* Device interfaces. */ 293d49fc192SJustin Hibbits static int 294d49fc192SJustin Hibbits xive_probe(device_t dev) 295d49fc192SJustin Hibbits { 296d49fc192SJustin Hibbits 297d49fc192SJustin Hibbits if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-pe")) 298d49fc192SJustin Hibbits return (ENXIO); 299d49fc192SJustin Hibbits 300d49fc192SJustin Hibbits device_set_desc(dev, "External Interrupt Virtualization Engine"); 301d49fc192SJustin Hibbits 302d49fc192SJustin Hibbits /* Make sure we always win against the xicp driver. */ 303d49fc192SJustin Hibbits return (BUS_PROBE_DEFAULT); 304d49fc192SJustin Hibbits } 305d49fc192SJustin Hibbits 306d49fc192SJustin Hibbits static int 307d49fc192SJustin Hibbits xics_probe(device_t dev) 308d49fc192SJustin Hibbits { 309d49fc192SJustin Hibbits 310d49fc192SJustin Hibbits if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-vc")) 311d49fc192SJustin Hibbits return (ENXIO); 312d49fc192SJustin Hibbits 313d49fc192SJustin Hibbits device_set_desc(dev, "External Interrupt Virtualization Engine Root"); 314d49fc192SJustin Hibbits return (BUS_PROBE_DEFAULT); 315d49fc192SJustin Hibbits } 316d49fc192SJustin Hibbits 317d49fc192SJustin Hibbits static int 318d49fc192SJustin Hibbits xive_attach(device_t dev) 319d49fc192SJustin Hibbits { 320d49fc192SJustin Hibbits struct xive_softc *sc = device_get_softc(dev); 321d49fc192SJustin Hibbits struct xive_cpu *xive_cpud; 322d49fc192SJustin Hibbits phandle_t phandle = ofw_bus_get_node(dev); 323d49fc192SJustin Hibbits int64_t vp_block; 324d49fc192SJustin Hibbits int error; 325d49fc192SJustin Hibbits int rid; 326d49fc192SJustin Hibbits int i, order; 327d49fc192SJustin Hibbits uint64_t vp_id; 328d49fc192SJustin Hibbits int64_t ipi_irq; 329d49fc192SJustin Hibbits 330d49fc192SJustin Hibbits opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EXP); 331d49fc192SJustin Hibbits 332d49fc192SJustin Hibbits error = OF_getencprop(phandle, "ibm,xive-provision-page-size", 333d49fc192SJustin Hibbits (pcell_t *)&sc->sc_prov_page_size, sizeof(sc->sc_prov_page_size)); 334d49fc192SJustin Hibbits 335d49fc192SJustin Hibbits rid = 1; /* Get the Hypervisor-level register set. */ 336d49fc192SJustin Hibbits sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 337d49fc192SJustin Hibbits &rid, RF_ACTIVE); 338d49fc192SJustin Hibbits sc->sc_offset = XIVE_TM_QW3_HV; 339d49fc192SJustin Hibbits 340d49fc192SJustin Hibbits mtx_init(&sc->sc_mtx, "XIVE", NULL, MTX_DEF); 341d49fc192SJustin Hibbits 34269576451SBrandon Bergren /* Workaround for qemu single-thread powernv */ 34369576451SBrandon Bergren if (mp_maxid == 0) 34469576451SBrandon Bergren order = 1; 34569576451SBrandon Bergren else 346d49fc192SJustin Hibbits order = fls(mp_maxid + (mp_maxid - 1)) - 1; 347d49fc192SJustin Hibbits 348d49fc192SJustin Hibbits do { 349d49fc192SJustin Hibbits vp_block = opal_call(OPAL_XIVE_ALLOCATE_VP_BLOCK, order); 350d49fc192SJustin Hibbits if (vp_block == OPAL_BUSY) 351d49fc192SJustin Hibbits DELAY(10); 352d49fc192SJustin Hibbits else if (vp_block == OPAL_XIVE_PROVISIONING) 353d49fc192SJustin Hibbits xive_provision_page(sc); 354d49fc192SJustin Hibbits else 355d49fc192SJustin Hibbits break; 356d49fc192SJustin Hibbits } while (1); 357d49fc192SJustin Hibbits 358d49fc192SJustin Hibbits if (vp_block < 0) { 359d49fc192SJustin Hibbits device_printf(dev, 360d49fc192SJustin Hibbits "Unable to allocate VP block. Opal error %d\n", 361d49fc192SJustin Hibbits (int)vp_block); 362d49fc192SJustin Hibbits bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem); 363d49fc192SJustin Hibbits return (ENXIO); 364d49fc192SJustin Hibbits } 365d49fc192SJustin Hibbits 366d49fc192SJustin Hibbits /* 367d49fc192SJustin Hibbits * Set up the VPs. Try to do as much as we can in attach, to lessen 368d49fc192SJustin Hibbits * what's needed at AP spawn time. 369d49fc192SJustin Hibbits */ 370d49fc192SJustin Hibbits CPU_FOREACH(i) { 371d49fc192SJustin Hibbits vp_id = pcpu_find(i)->pc_hwref; 372d49fc192SJustin Hibbits 373d49fc192SJustin Hibbits xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data); 374d49fc192SJustin Hibbits xive_cpud->vp = vp_id + vp_block; 375d49fc192SJustin Hibbits opal_call(OPAL_XIVE_GET_VP_INFO, xive_cpud->vp, NULL, 376d49fc192SJustin Hibbits vtophys(&xive_cpud->cam), NULL, vtophys(&xive_cpud->chip)); 377d49fc192SJustin Hibbits 378f9acb7a8SBrandon Bergren xive_cpud->cam = be64toh(xive_cpud->cam); 379f9acb7a8SBrandon Bergren xive_cpud->chip = be64toh(xive_cpud->chip); 380f9acb7a8SBrandon Bergren 381d49fc192SJustin Hibbits /* Allocate the queue page and populate the queue state data. */ 382d49fc192SJustin Hibbits xive_cpud->queue.q_page = contigmalloc(PAGE_SIZE, M_XIVE, 383d49fc192SJustin Hibbits M_ZERO | M_WAITOK, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 384d49fc192SJustin Hibbits xive_cpud->queue.q_size = 1 << PAGE_SHIFT; 385d49fc192SJustin Hibbits xive_cpud->queue.q_mask = 386d49fc192SJustin Hibbits ((xive_cpud->queue.q_size / sizeof(int)) - 1); 387d49fc192SJustin Hibbits xive_cpud->queue.q_toggle = 0; 388d49fc192SJustin Hibbits xive_cpud->queue.q_index = 0; 389d49fc192SJustin Hibbits do { 390d49fc192SJustin Hibbits error = opal_call(OPAL_XIVE_SET_VP_INFO, xive_cpud->vp, 391d49fc192SJustin Hibbits OPAL_XIVE_VP_ENABLED, 0); 392d49fc192SJustin Hibbits } while (error == OPAL_BUSY); 393d49fc192SJustin Hibbits error = opal_call(OPAL_XIVE_SET_QUEUE_INFO, vp_id, 394d49fc192SJustin Hibbits XIVE_PRIORITY, vtophys(xive_cpud->queue.q_page), PAGE_SHIFT, 395d49fc192SJustin Hibbits OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED); 396d49fc192SJustin Hibbits 397d49fc192SJustin Hibbits do { 398d49fc192SJustin Hibbits ipi_irq = opal_call(OPAL_XIVE_ALLOCATE_IRQ, 399d49fc192SJustin Hibbits xive_cpud->chip); 400d49fc192SJustin Hibbits } while (ipi_irq == OPAL_BUSY); 401d49fc192SJustin Hibbits 402d49fc192SJustin Hibbits if (ipi_irq < 0) 403d49fc192SJustin Hibbits device_printf(root_pic, 404d49fc192SJustin Hibbits "Failed allocating IPI. OPAL error %d\n", 405d49fc192SJustin Hibbits (int)ipi_irq); 406d49fc192SJustin Hibbits else { 407d49fc192SJustin Hibbits xive_init_irq(&xive_cpud->ipi_data, ipi_irq); 408d49fc192SJustin Hibbits xive_cpud->ipi_data.vp = vp_id; 409d49fc192SJustin Hibbits xive_cpud->ipi_data.lirq = MAX_XIVE_IRQS; 410d49fc192SJustin Hibbits opal_call(OPAL_XIVE_SET_IRQ_CONFIG, ipi_irq, 411d49fc192SJustin Hibbits xive_cpud->ipi_data.vp, XIVE_PRIORITY, 412d49fc192SJustin Hibbits MAX_XIVE_IRQS); 413d49fc192SJustin Hibbits } 414d49fc192SJustin Hibbits } 415d49fc192SJustin Hibbits 416d49fc192SJustin Hibbits powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XIVE_IRQS, 417d49fc192SJustin Hibbits 1 /* Number of IPIs */, FALSE); 418d49fc192SJustin Hibbits root_pic = dev; 419d49fc192SJustin Hibbits 420d49fc192SJustin Hibbits xive_setup_cpu(); 421d49fc192SJustin Hibbits powernv_smp_ap_extra_init = xive_smp_cpu_startup; 422d49fc192SJustin Hibbits 423d49fc192SJustin Hibbits return (0); 424d49fc192SJustin Hibbits } 425d49fc192SJustin Hibbits 426d49fc192SJustin Hibbits static int 427d49fc192SJustin Hibbits xics_attach(device_t dev) 428d49fc192SJustin Hibbits { 429d49fc192SJustin Hibbits phandle_t phandle = ofw_bus_get_node(dev); 430d49fc192SJustin Hibbits 431d49fc192SJustin Hibbits /* The XIVE (root PIC) will handle all our interrupts */ 432d49fc192SJustin Hibbits powerpc_register_pic(root_pic, OF_xref_from_node(phandle), 433d49fc192SJustin Hibbits MAX_XIVE_IRQS, 1 /* Number of IPIs */, FALSE); 434d49fc192SJustin Hibbits 435d49fc192SJustin Hibbits return (0); 436d49fc192SJustin Hibbits } 437d49fc192SJustin Hibbits 438d49fc192SJustin Hibbits /* 439d49fc192SJustin Hibbits * PIC I/F methods. 440d49fc192SJustin Hibbits */ 441d49fc192SJustin Hibbits 442d49fc192SJustin Hibbits static void 443d49fc192SJustin Hibbits xive_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv) 444d49fc192SJustin Hibbits { 445d49fc192SJustin Hibbits struct xive_irq *irqd; 446d49fc192SJustin Hibbits int cpu; 447d49fc192SJustin Hibbits int ncpus, i, error; 448d49fc192SJustin Hibbits 449d49fc192SJustin Hibbits if (*priv == NULL) 450d49fc192SJustin Hibbits *priv = xive_configure_irq(irq); 451d49fc192SJustin Hibbits 452d49fc192SJustin Hibbits irqd = *priv; 453d49fc192SJustin Hibbits 454d49fc192SJustin Hibbits /* 455d49fc192SJustin Hibbits * This doesn't appear to actually support affinity groups, so pick a 456d49fc192SJustin Hibbits * random CPU. 457d49fc192SJustin Hibbits */ 458d49fc192SJustin Hibbits ncpus = 0; 459d49fc192SJustin Hibbits CPU_FOREACH(cpu) 460d49fc192SJustin Hibbits if (CPU_ISSET(cpu, &cpumask)) ncpus++; 461d49fc192SJustin Hibbits 462d49fc192SJustin Hibbits i = mftb() % ncpus; 463d49fc192SJustin Hibbits ncpus = 0; 464d49fc192SJustin Hibbits CPU_FOREACH(cpu) { 465d49fc192SJustin Hibbits if (!CPU_ISSET(cpu, &cpumask)) 466d49fc192SJustin Hibbits continue; 467d49fc192SJustin Hibbits if (ncpus == i) 468d49fc192SJustin Hibbits break; 469d49fc192SJustin Hibbits ncpus++; 470d49fc192SJustin Hibbits } 471d49fc192SJustin Hibbits 472b94b2fcdSBrandon Bergren opal_call(OPAL_XIVE_SYNC, OPAL_XIVE_SYNC_QUEUE, irq); 473d49fc192SJustin Hibbits 474d49fc192SJustin Hibbits irqd->vp = pcpu_find(cpu)->pc_hwref; 475d49fc192SJustin Hibbits error = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, irqd->vp, 476d49fc192SJustin Hibbits XIVE_PRIORITY, irqd->lirq); 477d49fc192SJustin Hibbits 478d49fc192SJustin Hibbits if (error < 0) 479d49fc192SJustin Hibbits panic("Cannot bind interrupt %d to CPU %d", irq, cpu); 480d49fc192SJustin Hibbits 481d49fc192SJustin Hibbits xive_eoi(dev, irq, irqd); 482d49fc192SJustin Hibbits } 483d49fc192SJustin Hibbits 484d49fc192SJustin Hibbits /* Read the next entry in the queue page and update the index. */ 485d49fc192SJustin Hibbits static int 486d49fc192SJustin Hibbits xive_read_eq(struct xive_queue *q) 487d49fc192SJustin Hibbits { 488d49fc192SJustin Hibbits uint32_t i = be32toh(q->q_page[q->q_index]); 489d49fc192SJustin Hibbits 490d49fc192SJustin Hibbits /* Check validity, using current queue polarity. */ 491d49fc192SJustin Hibbits if ((i >> 31) == q->q_toggle) 492d49fc192SJustin Hibbits return (0); 493d49fc192SJustin Hibbits 494d49fc192SJustin Hibbits q->q_index = (q->q_index + 1) & q->q_mask; 495d49fc192SJustin Hibbits 496d49fc192SJustin Hibbits if (q->q_index == 0) 497d49fc192SJustin Hibbits q->q_toggle ^= 1; 498d49fc192SJustin Hibbits 499d49fc192SJustin Hibbits return (i & 0x7fffffff); 500d49fc192SJustin Hibbits } 501d49fc192SJustin Hibbits 502d49fc192SJustin Hibbits static void 503d49fc192SJustin Hibbits xive_dispatch(device_t dev, struct trapframe *tf) 504d49fc192SJustin Hibbits { 505d49fc192SJustin Hibbits struct xive_softc *sc; 506d49fc192SJustin Hibbits struct xive_cpu *xive_cpud; 507d49fc192SJustin Hibbits uint32_t vector; 508d49fc192SJustin Hibbits uint16_t ack; 509d49fc192SJustin Hibbits uint8_t cppr, he; 510d49fc192SJustin Hibbits 511d49fc192SJustin Hibbits sc = device_get_softc(dev); 512d49fc192SJustin Hibbits 51360185d89SBrandon Bergren xive_cpud = DPCPU_PTR(xive_cpu_data); 514d49fc192SJustin Hibbits for (;;) { 515d49fc192SJustin Hibbits ack = xive_read_2(sc, XIVE_TM_SPC_ACK); 516d49fc192SJustin Hibbits cppr = (ack & 0xff); 517d49fc192SJustin Hibbits 518d49fc192SJustin Hibbits he = ack >> TM_QW3NSR_HE_SHIFT; 519d49fc192SJustin Hibbits 520d49fc192SJustin Hibbits if (he == TM_QW3_NSR_HE_NONE) 521d49fc192SJustin Hibbits break; 52260185d89SBrandon Bergren 52360185d89SBrandon Bergren else if (__predict_false(he != TM_QW3_NSR_HE_PHYS)) { 52460185d89SBrandon Bergren /* 52560185d89SBrandon Bergren * We don't support TM_QW3_NSR_HE_POOL or 52660185d89SBrandon Bergren * TM_QW3_NSR_HE_LSI interrupts. 52760185d89SBrandon Bergren */ 528d49fc192SJustin Hibbits device_printf(dev, 529d49fc192SJustin Hibbits "Unexpected interrupt he type: %d\n", he); 530d49fc192SJustin Hibbits goto end; 531d49fc192SJustin Hibbits } 532d49fc192SJustin Hibbits 533d49fc192SJustin Hibbits xive_write_1(sc, XIVE_TM_CPPR, cppr); 534d49fc192SJustin Hibbits 535d49fc192SJustin Hibbits for (;;) { 536d49fc192SJustin Hibbits vector = xive_read_eq(&xive_cpud->queue); 537d49fc192SJustin Hibbits 538d49fc192SJustin Hibbits if (vector == 0) 539d49fc192SJustin Hibbits break; 540d49fc192SJustin Hibbits 541d49fc192SJustin Hibbits if (vector == MAX_XIVE_IRQS) 542d49fc192SJustin Hibbits vector = xive_ipi_vector; 543d49fc192SJustin Hibbits 544d49fc192SJustin Hibbits powerpc_dispatch_intr(vector, tf); 545d49fc192SJustin Hibbits } 546d49fc192SJustin Hibbits } 547d49fc192SJustin Hibbits end: 548d49fc192SJustin Hibbits xive_write_1(sc, XIVE_TM_CPPR, 0xff); 549d49fc192SJustin Hibbits } 550d49fc192SJustin Hibbits 551d49fc192SJustin Hibbits static void 552d49fc192SJustin Hibbits xive_enable(device_t dev, u_int irq, u_int vector, void **priv) 553d49fc192SJustin Hibbits { 554d49fc192SJustin Hibbits struct xive_irq *irqd; 555d49fc192SJustin Hibbits cell_t status, cpu; 556d49fc192SJustin Hibbits 557d49fc192SJustin Hibbits if (irq == MAX_XIVE_IRQS) { 558d49fc192SJustin Hibbits if (xive_ipi_vector == -1) 559d49fc192SJustin Hibbits xive_ipi_vector = vector; 560d49fc192SJustin Hibbits return; 561d49fc192SJustin Hibbits } 562d49fc192SJustin Hibbits if (*priv == NULL) 563d49fc192SJustin Hibbits *priv = xive_configure_irq(irq); 564d49fc192SJustin Hibbits 565d49fc192SJustin Hibbits irqd = *priv; 566d49fc192SJustin Hibbits 567d49fc192SJustin Hibbits /* Bind to this CPU to start */ 568d49fc192SJustin Hibbits cpu = PCPU_GET(hwref); 569d49fc192SJustin Hibbits irqd->lirq = vector; 570d49fc192SJustin Hibbits 571d49fc192SJustin Hibbits for (;;) { 572d49fc192SJustin Hibbits status = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, cpu, 573d49fc192SJustin Hibbits XIVE_PRIORITY, vector); 574d49fc192SJustin Hibbits if (status != OPAL_BUSY) 575d49fc192SJustin Hibbits break; 576d49fc192SJustin Hibbits DELAY(10); 577d49fc192SJustin Hibbits } 578d49fc192SJustin Hibbits 579d49fc192SJustin Hibbits if (status != 0) 580d49fc192SJustin Hibbits panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq, 581d49fc192SJustin Hibbits cpu, status); 582d49fc192SJustin Hibbits 583d49fc192SJustin Hibbits xive_unmask(dev, irq, *priv); 584d49fc192SJustin Hibbits } 585d49fc192SJustin Hibbits 586d49fc192SJustin Hibbits static void 587d49fc192SJustin Hibbits xive_eoi(device_t dev, u_int irq, void *priv) 588d49fc192SJustin Hibbits { 589d49fc192SJustin Hibbits struct xive_irq *rirq; 590d49fc192SJustin Hibbits struct xive_cpu *cpud; 591d49fc192SJustin Hibbits uint8_t eoi_val; 592d49fc192SJustin Hibbits 593d49fc192SJustin Hibbits if (irq == MAX_XIVE_IRQS) { 594d49fc192SJustin Hibbits cpud = DPCPU_PTR(xive_cpu_data); 595d49fc192SJustin Hibbits rirq = &cpud->ipi_data; 596d49fc192SJustin Hibbits } else 597d49fc192SJustin Hibbits rirq = priv; 598d49fc192SJustin Hibbits 599*be48fe60SJustin Hibbits if (rirq->flags & OPAL_XIVE_IRQ_STORE_EOI) 600d49fc192SJustin Hibbits xive_write_mmap8(rirq->eoi_page + XIVE_IRQ_STORE_EOI, 0); 601d49fc192SJustin Hibbits else if (rirq->flags & OPAL_XIVE_IRQ_LSI) 602d49fc192SJustin Hibbits xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_LOAD_EOI); 603d49fc192SJustin Hibbits else { 604d49fc192SJustin Hibbits eoi_val = xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00); 605d49fc192SJustin Hibbits if ((eoi_val & XIVE_IRQ_VAL_Q) && rirq->trig_page != 0) 606d49fc192SJustin Hibbits xive_write_mmap8(rirq->trig_page, 0); 607d49fc192SJustin Hibbits } 608d49fc192SJustin Hibbits } 609d49fc192SJustin Hibbits 610d49fc192SJustin Hibbits static void 611d49fc192SJustin Hibbits xive_ipi(device_t dev, u_int cpu) 612d49fc192SJustin Hibbits { 613d49fc192SJustin Hibbits struct xive_cpu *xive_cpud; 614d49fc192SJustin Hibbits 615d49fc192SJustin Hibbits xive_cpud = DPCPU_ID_PTR(cpu, xive_cpu_data); 616d49fc192SJustin Hibbits 617d49fc192SJustin Hibbits if (xive_cpud->ipi_data.trig_page == 0) 618d49fc192SJustin Hibbits return; 619d49fc192SJustin Hibbits xive_write_mmap8(xive_cpud->ipi_data.trig_page, 0); 620d49fc192SJustin Hibbits } 621d49fc192SJustin Hibbits 622d49fc192SJustin Hibbits static void 623d49fc192SJustin Hibbits xive_mask(device_t dev, u_int irq, void *priv) 624d49fc192SJustin Hibbits { 625d49fc192SJustin Hibbits struct xive_irq *rirq; 626d49fc192SJustin Hibbits 627d49fc192SJustin Hibbits /* Never mask IPIs */ 628d49fc192SJustin Hibbits if (irq == MAX_XIVE_IRQS) 629d49fc192SJustin Hibbits return; 630d49fc192SJustin Hibbits 631d49fc192SJustin Hibbits rirq = priv; 632d49fc192SJustin Hibbits 633d49fc192SJustin Hibbits if (!(rirq->flags & OPAL_XIVE_IRQ_LSI)) 634d49fc192SJustin Hibbits return; 635d49fc192SJustin Hibbits xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_01); 636d49fc192SJustin Hibbits } 637d49fc192SJustin Hibbits 638d49fc192SJustin Hibbits static void 639d49fc192SJustin Hibbits xive_unmask(device_t dev, u_int irq, void *priv) 640d49fc192SJustin Hibbits { 641d49fc192SJustin Hibbits struct xive_irq *rirq; 642d49fc192SJustin Hibbits 643d49fc192SJustin Hibbits rirq = priv; 644d49fc192SJustin Hibbits 645d49fc192SJustin Hibbits xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00); 646d49fc192SJustin Hibbits } 647d49fc192SJustin Hibbits 648d49fc192SJustin Hibbits static void 649d49fc192SJustin Hibbits xive_translate_code(device_t dev, u_int irq, int code, 650d49fc192SJustin Hibbits enum intr_trigger *trig, enum intr_polarity *pol) 651d49fc192SJustin Hibbits { 652d49fc192SJustin Hibbits switch (code) { 653d49fc192SJustin Hibbits case 0: 654d49fc192SJustin Hibbits /* L to H edge */ 655d49fc192SJustin Hibbits *trig = INTR_TRIGGER_EDGE; 656d49fc192SJustin Hibbits *pol = INTR_POLARITY_HIGH; 657d49fc192SJustin Hibbits break; 658d49fc192SJustin Hibbits case 1: 659d49fc192SJustin Hibbits /* Active L level */ 660d49fc192SJustin Hibbits *trig = INTR_TRIGGER_LEVEL; 661d49fc192SJustin Hibbits *pol = INTR_POLARITY_LOW; 662d49fc192SJustin Hibbits break; 663d49fc192SJustin Hibbits default: 664d49fc192SJustin Hibbits *trig = INTR_TRIGGER_CONFORM; 665d49fc192SJustin Hibbits *pol = INTR_POLARITY_CONFORM; 666d49fc192SJustin Hibbits } 667d49fc192SJustin Hibbits } 668d49fc192SJustin Hibbits 669d49fc192SJustin Hibbits /* Private functions. */ 670d49fc192SJustin Hibbits /* 671d49fc192SJustin Hibbits * Setup the current CPU. Called by the BSP at driver attachment, and by each 672d49fc192SJustin Hibbits * AP at wakeup (via xive_smp_cpu_startup()). 673d49fc192SJustin Hibbits */ 674d49fc192SJustin Hibbits static void 675d49fc192SJustin Hibbits xive_setup_cpu(void) 676d49fc192SJustin Hibbits { 677d49fc192SJustin Hibbits struct xive_softc *sc; 678d49fc192SJustin Hibbits struct xive_cpu *cpup; 679d49fc192SJustin Hibbits uint32_t val; 680d49fc192SJustin Hibbits 681d49fc192SJustin Hibbits cpup = DPCPU_PTR(xive_cpu_data); 682d49fc192SJustin Hibbits 683d49fc192SJustin Hibbits sc = device_get_softc(root_pic); 684d49fc192SJustin Hibbits 685d49fc192SJustin Hibbits val = bus_read_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2); 686d49fc192SJustin Hibbits if (val & TM_QW2W2_VP) 687d49fc192SJustin Hibbits bus_read_8(sc->sc_mem, XIVE_TM_SPC_PULL_POOL_CTX); 688d49fc192SJustin Hibbits 689d49fc192SJustin Hibbits bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD0, 0xff); 690d49fc192SJustin Hibbits bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2, 691d49fc192SJustin Hibbits TM_QW2W2_VP | cpup->cam); 692d49fc192SJustin Hibbits 693d49fc192SJustin Hibbits xive_unmask(root_pic, cpup->ipi_data.girq, &cpup->ipi_data); 694d49fc192SJustin Hibbits xive_write_1(sc, XIVE_TM_CPPR, 0xff); 695d49fc192SJustin Hibbits } 696d49fc192SJustin Hibbits 697d49fc192SJustin Hibbits /* Populate an IRQ structure, mapping the EOI and trigger pages. */ 698d49fc192SJustin Hibbits static void 699d49fc192SJustin Hibbits xive_init_irq(struct xive_irq *irqd, u_int irq) 700d49fc192SJustin Hibbits { 701d49fc192SJustin Hibbits uint64_t eoi_phys, trig_phys; 702d49fc192SJustin Hibbits uint32_t esb_shift; 703d49fc192SJustin Hibbits 704d49fc192SJustin Hibbits opal_call(OPAL_XIVE_GET_IRQ_INFO, irq, 705d49fc192SJustin Hibbits vtophys(&irqd->flags), vtophys(&eoi_phys), 706d49fc192SJustin Hibbits vtophys(&trig_phys), vtophys(&esb_shift), 707d49fc192SJustin Hibbits vtophys(&irqd->chip)); 708d49fc192SJustin Hibbits 709f9acb7a8SBrandon Bergren irqd->flags = be64toh(irqd->flags); 710f9acb7a8SBrandon Bergren eoi_phys = be64toh(eoi_phys); 711f9acb7a8SBrandon Bergren trig_phys = be64toh(trig_phys); 712f9acb7a8SBrandon Bergren esb_shift = be32toh(esb_shift); 713f9acb7a8SBrandon Bergren irqd->chip = be32toh(irqd->chip); 714f9acb7a8SBrandon Bergren 715d49fc192SJustin Hibbits irqd->girq = irq; 716d49fc192SJustin Hibbits irqd->esb_size = 1 << esb_shift; 717d49fc192SJustin Hibbits irqd->eoi_page = (vm_offset_t)pmap_mapdev(eoi_phys, irqd->esb_size); 718d49fc192SJustin Hibbits 719d49fc192SJustin Hibbits if (eoi_phys == trig_phys) 720d49fc192SJustin Hibbits irqd->trig_page = irqd->eoi_page; 721d49fc192SJustin Hibbits else if (trig_phys != 0) 722d49fc192SJustin Hibbits irqd->trig_page = (vm_offset_t)pmap_mapdev(trig_phys, 723d49fc192SJustin Hibbits irqd->esb_size); 724d49fc192SJustin Hibbits else 725d49fc192SJustin Hibbits irqd->trig_page = 0; 726d49fc192SJustin Hibbits 727d49fc192SJustin Hibbits opal_call(OPAL_XIVE_GET_IRQ_CONFIG, irq, vtophys(&irqd->vp), 728d49fc192SJustin Hibbits vtophys(&irqd->prio), vtophys(&irqd->lirq)); 729f9acb7a8SBrandon Bergren 730f9acb7a8SBrandon Bergren irqd->vp = be64toh(irqd->vp); 731f9acb7a8SBrandon Bergren irqd->prio = be64toh(irqd->prio); 732f9acb7a8SBrandon Bergren irqd->lirq = be32toh(irqd->lirq); 733d49fc192SJustin Hibbits } 734d49fc192SJustin Hibbits 735d49fc192SJustin Hibbits /* Allocate an IRQ struct before populating it. */ 736d49fc192SJustin Hibbits static struct xive_irq * 737d49fc192SJustin Hibbits xive_configure_irq(u_int irq) 738d49fc192SJustin Hibbits { 739d49fc192SJustin Hibbits struct xive_irq *irqd; 740d49fc192SJustin Hibbits 741d49fc192SJustin Hibbits irqd = malloc(sizeof(struct xive_irq), M_XIVE, M_WAITOK); 742d49fc192SJustin Hibbits 743d49fc192SJustin Hibbits xive_init_irq(irqd, irq); 744d49fc192SJustin Hibbits 745d49fc192SJustin Hibbits return (irqd); 746d49fc192SJustin Hibbits } 747d49fc192SJustin Hibbits 748d49fc192SJustin Hibbits /* 749d49fc192SJustin Hibbits * Part of the OPAL API. OPAL_XIVE_ALLOCATE_VP_BLOCK might require more pages, 750d49fc192SJustin Hibbits * provisioned through this call. 751d49fc192SJustin Hibbits */ 752d49fc192SJustin Hibbits static int 753d49fc192SJustin Hibbits xive_provision_page(struct xive_softc *sc) 754d49fc192SJustin Hibbits { 755d49fc192SJustin Hibbits void *prov_page; 756d49fc192SJustin Hibbits int error; 757d49fc192SJustin Hibbits 758d49fc192SJustin Hibbits do { 759d49fc192SJustin Hibbits prov_page = contigmalloc(sc->sc_prov_page_size, M_XIVE, 0, 760d49fc192SJustin Hibbits 0, BUS_SPACE_MAXADDR, 761d49fc192SJustin Hibbits sc->sc_prov_page_size, sc->sc_prov_page_size); 762d49fc192SJustin Hibbits 763d49fc192SJustin Hibbits error = opal_call(OPAL_XIVE_DONATE_PAGE, -1, 764d49fc192SJustin Hibbits vtophys(prov_page)); 765d49fc192SJustin Hibbits } while (error == OPAL_XIVE_PROVISIONING); 766d49fc192SJustin Hibbits 767d49fc192SJustin Hibbits return (0); 768d49fc192SJustin Hibbits } 769d49fc192SJustin Hibbits 770d49fc192SJustin Hibbits /* The XIVE_TM_CPPR register must be set by each thread */ 771d49fc192SJustin Hibbits static void 772d49fc192SJustin Hibbits xive_smp_cpu_startup(void) 773d49fc192SJustin Hibbits { 774d49fc192SJustin Hibbits 775d49fc192SJustin Hibbits xive_setup_cpu(); 776d49fc192SJustin Hibbits } 777