xref: /freebsd/sys/powerpc/pseries/xics.c (revision 4d846d260e2b9a3d4d0a701462568268cbfe7a5b)
17a8d25c0SNathan Whitehorn /*-
2*4d846d26SWarner Losh  * SPDX-License-Identifier: BSD-2-Clause
371e3c308SPedro F. Giffuni  *
47a8d25c0SNathan Whitehorn  * Copyright 2011 Nathan Whitehorn
57a8d25c0SNathan Whitehorn  *
67a8d25c0SNathan Whitehorn  * Redistribution and use in source and binary forms, with or without
77a8d25c0SNathan Whitehorn  * modification, are permitted provided that the following conditions
87a8d25c0SNathan Whitehorn  * are met:
97a8d25c0SNathan Whitehorn  * 1. Redistributions of source code must retain the above copyright
107a8d25c0SNathan Whitehorn  *    notice, this list of conditions and the following disclaimer.
117a8d25c0SNathan Whitehorn  * 2. Redistributions in binary form must reproduce the above copyright
127a8d25c0SNathan Whitehorn  *    notice, this list of conditions and the following disclaimer in the
137a8d25c0SNathan Whitehorn  *    documentation and/or other materials provided with the distribution.
147a8d25c0SNathan Whitehorn  *
157a8d25c0SNathan Whitehorn  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
167a8d25c0SNathan Whitehorn  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
177a8d25c0SNathan Whitehorn  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
187a8d25c0SNathan Whitehorn  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
197a8d25c0SNathan Whitehorn  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
207a8d25c0SNathan Whitehorn  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
217a8d25c0SNathan Whitehorn  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
227a8d25c0SNathan Whitehorn  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
237a8d25c0SNathan Whitehorn  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
247a8d25c0SNathan Whitehorn  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
257a8d25c0SNathan Whitehorn  * SUCH DAMAGE.
267a8d25c0SNathan Whitehorn  */
277a8d25c0SNathan Whitehorn 
287a8d25c0SNathan Whitehorn #include <sys/cdefs.h>
297a8d25c0SNathan Whitehorn __FBSDID("$FreeBSD$");
307a8d25c0SNathan Whitehorn 
318fc8068eSWojciech Macek #include "opt_platform.h"
328fc8068eSWojciech Macek 
337a8d25c0SNathan Whitehorn #include <sys/param.h>
347a8d25c0SNathan Whitehorn #include <sys/systm.h>
357a8d25c0SNathan Whitehorn #include <sys/module.h>
367a8d25c0SNathan Whitehorn #include <sys/bus.h>
377a8d25c0SNathan Whitehorn #include <sys/conf.h>
387a8d25c0SNathan Whitehorn #include <sys/kernel.h>
39e2e050c8SConrad Meyer #include <sys/lock.h>
407a8d25c0SNathan Whitehorn #include <sys/malloc.h>
41e2e050c8SConrad Meyer #include <sys/mutex.h>
427a8d25c0SNathan Whitehorn #include <sys/smp.h>
437a8d25c0SNathan Whitehorn 
447a8d25c0SNathan Whitehorn #include <vm/vm.h>
457a8d25c0SNathan Whitehorn #include <vm/pmap.h>
467a8d25c0SNathan Whitehorn 
477a8d25c0SNathan Whitehorn #include <machine/bus.h>
487a8d25c0SNathan Whitehorn #include <machine/intr_machdep.h>
497a8d25c0SNathan Whitehorn #include <machine/md_var.h>
507a8d25c0SNathan Whitehorn #include <machine/rtas.h>
517a8d25c0SNathan Whitehorn 
527a8d25c0SNathan Whitehorn #include <dev/ofw/ofw_bus.h>
537a8d25c0SNathan Whitehorn #include <dev/ofw/ofw_bus_subr.h>
547a8d25c0SNathan Whitehorn 
558fc8068eSWojciech Macek #ifdef POWERNV
568fc8068eSWojciech Macek #include <powerpc/powernv/opal.h>
578fc8068eSWojciech Macek #endif
588fc8068eSWojciech Macek 
597a8d25c0SNathan Whitehorn #include "phyp-hvcall.h"
607a8d25c0SNathan Whitehorn #include "pic_if.h"
617a8d25c0SNathan Whitehorn 
627a8d25c0SNathan Whitehorn #define XICP_PRIORITY	5	/* Random non-zero number */
637a8d25c0SNathan Whitehorn #define XICP_IPI	2
647a8d25c0SNathan Whitehorn #define MAX_XICP_IRQS	(1<<24)	/* 24-bit XIRR field */
657a8d25c0SNathan Whitehorn 
667a8d25c0SNathan Whitehorn static int	xicp_probe(device_t);
677a8d25c0SNathan Whitehorn static int	xicp_attach(device_t);
687a8d25c0SNathan Whitehorn static int	xics_probe(device_t);
697a8d25c0SNathan Whitehorn static int	xics_attach(device_t);
707a8d25c0SNathan Whitehorn 
7156505ec0SJustin Hibbits static void	xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv);
727a8d25c0SNathan Whitehorn static void	xicp_dispatch(device_t, struct trapframe *);
7356505ec0SJustin Hibbits static void	xicp_enable(device_t, u_int, u_int, void **priv);
7456505ec0SJustin Hibbits static void	xicp_eoi(device_t, u_int, void *priv);
757a8d25c0SNathan Whitehorn static void	xicp_ipi(device_t, u_int);
7656505ec0SJustin Hibbits static void	xicp_mask(device_t, u_int, void *priv);
7756505ec0SJustin Hibbits static void	xicp_unmask(device_t, u_int, void *priv);
787a8d25c0SNathan Whitehorn 
79ef6da5e5SJustin Hibbits #ifdef POWERNV
80d49fc192SJustin Hibbits extern void (*powernv_smp_ap_extra_init)(void);
81d49fc192SJustin Hibbits static void	xicp_smp_cpu_startup(void);
82ef6da5e5SJustin Hibbits #endif
83ef6da5e5SJustin Hibbits 
847a8d25c0SNathan Whitehorn static device_method_t  xicp_methods[] = {
857a8d25c0SNathan Whitehorn 	/* Device interface */
867a8d25c0SNathan Whitehorn 	DEVMETHOD(device_probe,		xicp_probe),
877a8d25c0SNathan Whitehorn 	DEVMETHOD(device_attach,	xicp_attach),
887a8d25c0SNathan Whitehorn 
897a8d25c0SNathan Whitehorn 	/* PIC interface */
907a8d25c0SNathan Whitehorn 	DEVMETHOD(pic_bind,		xicp_bind),
917a8d25c0SNathan Whitehorn 	DEVMETHOD(pic_dispatch,		xicp_dispatch),
927a8d25c0SNathan Whitehorn 	DEVMETHOD(pic_enable,		xicp_enable),
937a8d25c0SNathan Whitehorn 	DEVMETHOD(pic_eoi,		xicp_eoi),
947a8d25c0SNathan Whitehorn 	DEVMETHOD(pic_ipi,		xicp_ipi),
957a8d25c0SNathan Whitehorn 	DEVMETHOD(pic_mask,		xicp_mask),
967a8d25c0SNathan Whitehorn 	DEVMETHOD(pic_unmask,		xicp_unmask),
977a8d25c0SNathan Whitehorn 
988fc8068eSWojciech Macek 	DEVMETHOD_END
997a8d25c0SNathan Whitehorn };
1007a8d25c0SNathan Whitehorn 
1017a8d25c0SNathan Whitehorn static device_method_t  xics_methods[] = {
1027a8d25c0SNathan Whitehorn 	/* Device interface */
1037a8d25c0SNathan Whitehorn 	DEVMETHOD(device_probe,		xics_probe),
1047a8d25c0SNathan Whitehorn 	DEVMETHOD(device_attach,	xics_attach),
1057a8d25c0SNathan Whitehorn 
1068fc8068eSWojciech Macek 	DEVMETHOD_END
1077a8d25c0SNathan Whitehorn };
1087a8d25c0SNathan Whitehorn 
10956505ec0SJustin Hibbits struct xicp_intvec {
11056505ec0SJustin Hibbits 	int irq;
11156505ec0SJustin Hibbits 	int vector;
11256505ec0SJustin Hibbits 	int cpu;
11356505ec0SJustin Hibbits };
11456505ec0SJustin Hibbits 
1157a8d25c0SNathan Whitehorn struct xicp_softc {
1167a8d25c0SNathan Whitehorn 	struct mtx sc_mtx;
1178fc8068eSWojciech Macek 	struct resource *mem[MAXCPU];
1188fc8068eSWojciech Macek 
1198fc8068eSWojciech Macek 	int cpu_range[2];
1207a8d25c0SNathan Whitehorn 
1217a8d25c0SNathan Whitehorn 	int ibm_int_on;
1227a8d25c0SNathan Whitehorn 	int ibm_int_off;
1237a8d25c0SNathan Whitehorn 	int ibm_get_xive;
1247a8d25c0SNathan Whitehorn 	int ibm_set_xive;
1257a8d25c0SNathan Whitehorn 
1267a8d25c0SNathan Whitehorn 	/* XXX: inefficient -- hash table? tree? */
12756505ec0SJustin Hibbits 	struct xicp_intvec intvecs[256];
1287a8d25c0SNathan Whitehorn 	int nintvecs;
129431d31e0SJustin Hibbits 	int ipi_vec;
130ef6da5e5SJustin Hibbits 	bool xics_emu;
1317a8d25c0SNathan Whitehorn };
1327a8d25c0SNathan Whitehorn 
1337a8d25c0SNathan Whitehorn static driver_t xicp_driver = {
1347a8d25c0SNathan Whitehorn 	"xicp",
1357a8d25c0SNathan Whitehorn 	xicp_methods,
1367a8d25c0SNathan Whitehorn 	sizeof(struct xicp_softc)
1377a8d25c0SNathan Whitehorn };
1387a8d25c0SNathan Whitehorn 
1397a8d25c0SNathan Whitehorn static driver_t xics_driver = {
1407a8d25c0SNathan Whitehorn 	"xics",
1417a8d25c0SNathan Whitehorn 	xics_methods,
1427a8d25c0SNathan Whitehorn 	0
1437a8d25c0SNathan Whitehorn };
1447a8d25c0SNathan Whitehorn 
1456cff19a3SNathan Whitehorn #ifdef POWERNV
1465272c9bdSJustin Hibbits /* We can only pass physical addresses into OPAL.  Kernel stacks are in the KVA,
1475272c9bdSJustin Hibbits  * not in the direct map, so we need to somehow extract the physical address.
1485272c9bdSJustin Hibbits  * However, pmap_kextract() takes locks, which is forbidden in a critical region
14954b310b8SJustin Hibbits  * (which PIC_DISPATCH() operates in).  The kernel is mapped into the Direct
1505272c9bdSJustin Hibbits  * Map (0xc000....), and the CPU implicitly drops the top two bits when doing
1515272c9bdSJustin Hibbits  * real address by nature that the bus width is smaller than 64-bits.  Placing
1525272c9bdSJustin Hibbits  * cpu_xirr into the DMAP lets us take advantage of this and avoids the
1535272c9bdSJustin Hibbits  * pmap_kextract() that would otherwise be needed if using the stack variable.
1545272c9bdSJustin Hibbits  */
155ef6da5e5SJustin Hibbits static uint32_t cpu_xirr[MAXCPU];
1566cff19a3SNathan Whitehorn #endif
157ef6da5e5SJustin Hibbits 
1589b9a5327SJohn Baldwin EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, 0, 0, BUS_PASS_INTERRUPT - 1);
1599b9a5327SJohn Baldwin EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, 0, 0, BUS_PASS_INTERRUPT);
1607a8d25c0SNathan Whitehorn 
1618fc8068eSWojciech Macek #ifdef POWERNV
1628fc8068eSWojciech Macek static struct resource *
1638fc8068eSWojciech Macek xicp_mem_for_cpu(int cpu)
1648fc8068eSWojciech Macek {
165e4a38f54SJohn Baldwin 	devclass_t dc;
1668fc8068eSWojciech Macek 	device_t dev;
1678fc8068eSWojciech Macek 	struct xicp_softc *sc;
1688fc8068eSWojciech Macek 	int i;
1698fc8068eSWojciech Macek 
170e4a38f54SJohn Baldwin 	dc = devclass_find(xicp_driver.name);
171e4a38f54SJohn Baldwin 	for (i = 0; (dev = devclass_get_device(dc, i)) != NULL; i++){
1728fc8068eSWojciech Macek 		sc = device_get_softc(dev);
1738fc8068eSWojciech Macek 		if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1])
1748fc8068eSWojciech Macek 			return (sc->mem[cpu - sc->cpu_range[0]]);
1758fc8068eSWojciech Macek 	}
1768fc8068eSWojciech Macek 
1778fc8068eSWojciech Macek 	return (NULL);
1788fc8068eSWojciech Macek }
1798fc8068eSWojciech Macek #endif
1808fc8068eSWojciech Macek 
1817a8d25c0SNathan Whitehorn static int
1827a8d25c0SNathan Whitehorn xicp_probe(device_t dev)
1837a8d25c0SNathan Whitehorn {
1847a8d25c0SNathan Whitehorn 
185ef6da5e5SJustin Hibbits 	if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") &&
186ef6da5e5SJustin Hibbits 	    !ofw_bus_is_compatible(dev, "ibm,opal-intc"))
1877a8d25c0SNathan Whitehorn 		return (ENXIO);
1887a8d25c0SNathan Whitehorn 
1898fc8068eSWojciech Macek 	device_set_desc(dev, "External Interrupt Presentation Controller");
1907a8d25c0SNathan Whitehorn 	return (BUS_PROBE_GENERIC);
1917a8d25c0SNathan Whitehorn }
1927a8d25c0SNathan Whitehorn 
1937a8d25c0SNathan Whitehorn static int
1947a8d25c0SNathan Whitehorn xics_probe(device_t dev)
1957a8d25c0SNathan Whitehorn {
1967a8d25c0SNathan Whitehorn 
197ef6da5e5SJustin Hibbits 	if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") &&
198ef6da5e5SJustin Hibbits 	    !ofw_bus_is_compatible(dev, "IBM,opal-xics"))
1997a8d25c0SNathan Whitehorn 		return (ENXIO);
2007a8d25c0SNathan Whitehorn 
2018fc8068eSWojciech Macek 	device_set_desc(dev, "External Interrupt Source Controller");
2027a8d25c0SNathan Whitehorn 	return (BUS_PROBE_GENERIC);
2037a8d25c0SNathan Whitehorn }
2047a8d25c0SNathan Whitehorn 
2057a8d25c0SNathan Whitehorn static int
2067a8d25c0SNathan Whitehorn xicp_attach(device_t dev)
2077a8d25c0SNathan Whitehorn {
2087a8d25c0SNathan Whitehorn 	struct xicp_softc *sc = device_get_softc(dev);
2097a8d25c0SNathan Whitehorn 	phandle_t phandle = ofw_bus_get_node(dev);
2107a8d25c0SNathan Whitehorn 
2118fc8068eSWojciech Macek 	if (rtas_exists()) {
2127a8d25c0SNathan Whitehorn 		sc->ibm_int_on = rtas_token_lookup("ibm,int-on");
2137a8d25c0SNathan Whitehorn 		sc->ibm_int_off = rtas_token_lookup("ibm,int-off");
2147a8d25c0SNathan Whitehorn 		sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive");
2157a8d25c0SNathan Whitehorn 		sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive");
2168fc8068eSWojciech Macek #ifdef POWERNV
2178fc8068eSWojciech Macek 	} else if (opal_check() == 0) {
2188fc8068eSWojciech Macek 		/* No init needed */
2198fc8068eSWojciech Macek #endif
2208fc8068eSWojciech Macek 	} else {
2218fc8068eSWojciech Macek 		device_printf(dev, "Cannot attach without RTAS or OPAL\n");
2228fc8068eSWojciech Macek 		return (ENXIO);
2238fc8068eSWojciech Macek 	}
2248fc8068eSWojciech Macek 
2258fc8068eSWojciech Macek 	if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) {
2268fc8068eSWojciech Macek 		OF_getencprop(phandle, "ibm,interrupt-server-ranges",
2278fc8068eSWojciech Macek 		    sc->cpu_range, sizeof(sc->cpu_range));
2288fc8068eSWojciech Macek 		sc->cpu_range[1] += sc->cpu_range[0];
2298fc8068eSWojciech Macek 		device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0],
2308fc8068eSWojciech Macek 		    sc->cpu_range[1]-1);
231ef6da5e5SJustin Hibbits #ifdef POWERNV
232ef6da5e5SJustin Hibbits 	} else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) {
233ef6da5e5SJustin Hibbits 			/*
234ef6da5e5SJustin Hibbits 			 * For now run POWER9 XIVE interrupt controller in XICS
235ef6da5e5SJustin Hibbits 			 * compatibility mode.
236ef6da5e5SJustin Hibbits 			 */
237ef6da5e5SJustin Hibbits 			sc->xics_emu = true;
238d49fc192SJustin Hibbits 			opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EMU);
239ef6da5e5SJustin Hibbits #endif
2408fc8068eSWojciech Macek 	} else {
2418fc8068eSWojciech Macek 		sc->cpu_range[0] = 0;
2428fc8068eSWojciech Macek 		sc->cpu_range[1] = mp_ncpus;
2438fc8068eSWojciech Macek 	}
2448fc8068eSWojciech Macek 
2458fc8068eSWojciech Macek #ifdef POWERNV
2468fc8068eSWojciech Macek 	if (mfmsr() & PSL_HV) {
2478fc8068eSWojciech Macek 		int i;
2488fc8068eSWojciech Macek 
249ef6da5e5SJustin Hibbits 		if (sc->xics_emu) {
250ef6da5e5SJustin Hibbits 			opal_call(OPAL_INT_SET_CPPR, 0xff);
251ef6da5e5SJustin Hibbits 			for (i = 0; i < mp_ncpus; i++) {
252ef6da5e5SJustin Hibbits 				opal_call(OPAL_INT_SET_MFRR,
253ef6da5e5SJustin Hibbits 				    pcpu_find(i)->pc_hwref, 0xff);
254ef6da5e5SJustin Hibbits 			}
255ef6da5e5SJustin Hibbits 		} else {
2568fc8068eSWojciech Macek 			for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) {
2578fc8068eSWojciech Macek 				sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
2588fc8068eSWojciech Macek 				    &i, RF_ACTIVE);
2598fc8068eSWojciech Macek 				if (sc->mem[i] == NULL) {
2608fc8068eSWojciech Macek 					device_printf(dev, "Could not alloc mem "
2618fc8068eSWojciech Macek 					    "resource %d\n", i);
2628fc8068eSWojciech Macek 					return (ENXIO);
2638fc8068eSWojciech Macek 				}
2648fc8068eSWojciech Macek 
2658fc8068eSWojciech Macek 				/* Unmask interrupts on all cores */
2668fc8068eSWojciech Macek 				bus_write_1(sc->mem[i], 4, 0xff);
2678fc8068eSWojciech Macek 				bus_write_1(sc->mem[i], 12, 0xff);
2688fc8068eSWojciech Macek 			}
2698fc8068eSWojciech Macek 		}
270ef6da5e5SJustin Hibbits 	}
2718fc8068eSWojciech Macek #endif
2728fc8068eSWojciech Macek 
2738fc8068eSWojciech Macek 	mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF);
2748fc8068eSWojciech Macek 	sc->nintvecs = 0;
2757a8d25c0SNathan Whitehorn 
27644d29d47SNathan Whitehorn 	powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS,
2777a8d25c0SNathan Whitehorn 	    1 /* Number of IPIs */, FALSE);
2787a8d25c0SNathan Whitehorn 	root_pic = dev;
2797a8d25c0SNathan Whitehorn 
280d49fc192SJustin Hibbits #ifdef POWERNV
281d49fc192SJustin Hibbits 	if (sc->xics_emu)
282d49fc192SJustin Hibbits 		powernv_smp_ap_extra_init = xicp_smp_cpu_startup;
283d49fc192SJustin Hibbits #endif
284d49fc192SJustin Hibbits 
2857a8d25c0SNathan Whitehorn 	return (0);
2867a8d25c0SNathan Whitehorn }
2877a8d25c0SNathan Whitehorn 
2887a8d25c0SNathan Whitehorn static int
2897a8d25c0SNathan Whitehorn xics_attach(device_t dev)
2907a8d25c0SNathan Whitehorn {
2917a8d25c0SNathan Whitehorn 	phandle_t phandle = ofw_bus_get_node(dev);
2927a8d25c0SNathan Whitehorn 
2937a8d25c0SNathan Whitehorn 	/* The XICP (root PIC) will handle all our interrupts */
29444d29d47SNathan Whitehorn 	powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
29544d29d47SNathan Whitehorn 	    MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE);
2967a8d25c0SNathan Whitehorn 
2977a8d25c0SNathan Whitehorn 	return (0);
2987a8d25c0SNathan Whitehorn }
2997a8d25c0SNathan Whitehorn 
30015fba9d3SJustin Hibbits static __inline struct xicp_intvec *
30115fba9d3SJustin Hibbits xicp_setup_priv(struct xicp_softc *sc, u_int irq, void **priv)
30215fba9d3SJustin Hibbits {
30315fba9d3SJustin Hibbits 	if (*priv == NULL) {
30415fba9d3SJustin Hibbits 		KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs),
30515fba9d3SJustin Hibbits 			("Too many XICP interrupts"));
30615fba9d3SJustin Hibbits 		mtx_lock(&sc->sc_mtx);
30715fba9d3SJustin Hibbits 		*priv = &sc->intvecs[sc->nintvecs++];
30815fba9d3SJustin Hibbits 		mtx_unlock(&sc->sc_mtx);
30915fba9d3SJustin Hibbits 	}
31015fba9d3SJustin Hibbits 
31115fba9d3SJustin Hibbits 	return (*priv);
31215fba9d3SJustin Hibbits }
31315fba9d3SJustin Hibbits 
3147a8d25c0SNathan Whitehorn /*
3157a8d25c0SNathan Whitehorn  * PIC I/F methods.
3167a8d25c0SNathan Whitehorn  */
3177a8d25c0SNathan Whitehorn 
3187a8d25c0SNathan Whitehorn static void
31956505ec0SJustin Hibbits xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
3207a8d25c0SNathan Whitehorn {
3217a8d25c0SNathan Whitehorn 	struct xicp_softc *sc = device_get_softc(dev);
32256505ec0SJustin Hibbits 	struct xicp_intvec *iv;
3237a8d25c0SNathan Whitehorn 	cell_t status, cpu;
32449f10b51SLeandro Lupori 	int ncpus, i, error = -1;
3257a8d25c0SNathan Whitehorn 
326f0393bbfSWojciech Macek 	/* Ignore IPIs */
327f0393bbfSWojciech Macek 	if (irq == MAX_XICP_IRQS)
328f0393bbfSWojciech Macek 		return;
329f0393bbfSWojciech Macek 
33015fba9d3SJustin Hibbits 	iv = xicp_setup_priv(sc, irq, priv);
33156505ec0SJustin Hibbits 
3327a8d25c0SNathan Whitehorn 	/*
3330174acd4SNathan Whitehorn 	 * This doesn't appear to actually support affinity groups, so pick a
3340174acd4SNathan Whitehorn 	 * random CPU.
3357a8d25c0SNathan Whitehorn 	 */
336a4c6f6e5SNathan Whitehorn 	ncpus = 0;
3377a8d25c0SNathan Whitehorn 	CPU_FOREACH(cpu)
3380174acd4SNathan Whitehorn 		if (CPU_ISSET(cpu, &cpumask)) ncpus++;
3390174acd4SNathan Whitehorn 
3400174acd4SNathan Whitehorn 	i = mftb() % ncpus;
3410174acd4SNathan Whitehorn 	ncpus = 0;
3420174acd4SNathan Whitehorn 	CPU_FOREACH(cpu) {
3430174acd4SNathan Whitehorn 		if (!CPU_ISSET(cpu, &cpumask))
3440174acd4SNathan Whitehorn 			continue;
3450174acd4SNathan Whitehorn 		if (ncpus == i)
3460174acd4SNathan Whitehorn 			break;
3470174acd4SNathan Whitehorn 		ncpus++;
3480174acd4SNathan Whitehorn 	}
3490174acd4SNathan Whitehorn 
350f0393bbfSWojciech Macek 	cpu = pcpu_find(cpu)->pc_hwref;
35156505ec0SJustin Hibbits 	iv->cpu = cpu;
3527a8d25c0SNathan Whitehorn 
3538fc8068eSWojciech Macek 	if (rtas_exists())
354a4c6f6e5SNathan Whitehorn 		error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
355a4c6f6e5SNathan Whitehorn 		    XICP_PRIORITY, &status);
3568fc8068eSWojciech Macek #ifdef POWERNV
3578fc8068eSWojciech Macek 	else
3588fc8068eSWojciech Macek 		error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
3598fc8068eSWojciech Macek #endif
3608fc8068eSWojciech Macek 
361a4c6f6e5SNathan Whitehorn 	if (error < 0)
362a4c6f6e5SNathan Whitehorn 		panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
3637a8d25c0SNathan Whitehorn }
3647a8d25c0SNathan Whitehorn 
3657a8d25c0SNathan Whitehorn static void
3667a8d25c0SNathan Whitehorn xicp_dispatch(device_t dev, struct trapframe *tf)
3677a8d25c0SNathan Whitehorn {
3687a8d25c0SNathan Whitehorn 	struct xicp_softc *sc;
3698fc8068eSWojciech Macek 	struct resource *regs = NULL;
3707a8d25c0SNathan Whitehorn 	uint64_t xirr, junk;
3717a8d25c0SNathan Whitehorn 	int i;
3727a8d25c0SNathan Whitehorn 
373ef6da5e5SJustin Hibbits 	sc = device_get_softc(dev);
3748fc8068eSWojciech Macek #ifdef POWERNV
375ef6da5e5SJustin Hibbits 	if ((mfmsr() & PSL_HV) && !sc->xics_emu) {
376f0393bbfSWojciech Macek 		regs = xicp_mem_for_cpu(PCPU_GET(hwref));
3778fc8068eSWojciech Macek 		KASSERT(regs != NULL,
378f0393bbfSWojciech Macek 		    ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref)));
3798fc8068eSWojciech Macek 	}
3808fc8068eSWojciech Macek #endif
3818fc8068eSWojciech Macek 
3827a8d25c0SNathan Whitehorn 	for (;;) {
3837a8d25c0SNathan Whitehorn 		/* Return value in R4, use the PFT call */
3848fc8068eSWojciech Macek 		if (regs) {
3858fc8068eSWojciech Macek 			xirr = bus_read_4(regs, 4);
386ef6da5e5SJustin Hibbits #ifdef POWERNV
387ef6da5e5SJustin Hibbits 		} else if (sc->xics_emu) {
388ef6da5e5SJustin Hibbits 			opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)],
389ef6da5e5SJustin Hibbits 			    false);
390ef6da5e5SJustin Hibbits 			xirr = cpu_xirr[PCPU_GET(cpuid)];
391ef6da5e5SJustin Hibbits #endif
3928fc8068eSWojciech Macek 		} else {
3938fc8068eSWojciech Macek 			/* Return value in R4, use the PFT call */
3947a8d25c0SNathan Whitehorn 			phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk);
3958fc8068eSWojciech Macek 		}
3967a8d25c0SNathan Whitehorn 		xirr &= 0x00ffffff;
3977a8d25c0SNathan Whitehorn 
3987e524b07SJustin Hibbits 		if (xirr == 0) /* No more pending interrupts? */
3997a8d25c0SNathan Whitehorn 			break;
4007e524b07SJustin Hibbits 
4017a8d25c0SNathan Whitehorn 		if (xirr == XICP_IPI) {		/* Magic number for IPIs */
4027a8d25c0SNathan Whitehorn 			xirr = MAX_XICP_IRQS;	/* Map to FreeBSD magic */
4038fc8068eSWojciech Macek 
4048fc8068eSWojciech Macek 			/* Clear IPI */
4058fc8068eSWojciech Macek 			if (regs)
4068fc8068eSWojciech Macek 				bus_write_1(regs, 12, 0xff);
407ef6da5e5SJustin Hibbits #ifdef POWERNV
408ef6da5e5SJustin Hibbits 			else if (sc->xics_emu)
409ef6da5e5SJustin Hibbits 				opal_call(OPAL_INT_SET_MFRR,
410ef6da5e5SJustin Hibbits 				    PCPU_GET(hwref), 0xff);
411ef6da5e5SJustin Hibbits #endif
4128fc8068eSWojciech Macek 			else
413f0393bbfSWojciech Macek 				phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)),
4148fc8068eSWojciech Macek 				    0xff);
415431d31e0SJustin Hibbits 			i = sc->ipi_vec;
416431d31e0SJustin Hibbits 		} else {
4177a8d25c0SNathan Whitehorn 			/* XXX: super inefficient */
4187a8d25c0SNathan Whitehorn 			for (i = 0; i < sc->nintvecs; i++) {
4197a8d25c0SNathan Whitehorn 				if (sc->intvecs[i].irq == xirr)
4207a8d25c0SNathan Whitehorn 					break;
4217a8d25c0SNathan Whitehorn 			}
4227a8d25c0SNathan Whitehorn 			KASSERT(i < sc->nintvecs, ("Unmapped XIRR"));
423431d31e0SJustin Hibbits 		}
424431d31e0SJustin Hibbits 
4257a8d25c0SNathan Whitehorn 		powerpc_dispatch_intr(sc->intvecs[i].vector, tf);
4267a8d25c0SNathan Whitehorn 	}
4277a8d25c0SNathan Whitehorn }
4287a8d25c0SNathan Whitehorn 
4297a8d25c0SNathan Whitehorn static void
43056505ec0SJustin Hibbits xicp_enable(device_t dev, u_int irq, u_int vector, void **priv)
4317a8d25c0SNathan Whitehorn {
4327a8d25c0SNathan Whitehorn 	struct xicp_softc *sc;
43356505ec0SJustin Hibbits 	struct xicp_intvec *intr;
4347a8d25c0SNathan Whitehorn 	cell_t status, cpu;
4357a8d25c0SNathan Whitehorn 
4367a8d25c0SNathan Whitehorn 	sc = device_get_softc(dev);
4377a8d25c0SNathan Whitehorn 
4388fc8068eSWojciech Macek 	/* Bind to this CPU to start: distrib. ID is last entry in gserver# */
439f0393bbfSWojciech Macek 	cpu = PCPU_GET(hwref);
4408fc8068eSWojciech Macek 
44115fba9d3SJustin Hibbits 	intr = xicp_setup_priv(sc, irq, priv);
44256505ec0SJustin Hibbits 
44356505ec0SJustin Hibbits 	intr->irq = irq;
44456505ec0SJustin Hibbits 	intr->vector = vector;
44556505ec0SJustin Hibbits 	intr->cpu = cpu;
44656505ec0SJustin Hibbits 	mb();
4477a8d25c0SNathan Whitehorn 
448431d31e0SJustin Hibbits 	/* IPIs are also enabled.  Stash off the vector index */
449431d31e0SJustin Hibbits 	if (irq == MAX_XICP_IRQS) {
450431d31e0SJustin Hibbits 		sc->ipi_vec = intr - sc->intvecs;
4517a8d25c0SNathan Whitehorn 		return;
452431d31e0SJustin Hibbits 	}
4537a8d25c0SNathan Whitehorn 
4548fc8068eSWojciech Macek 	if (rtas_exists()) {
4558fc8068eSWojciech Macek 		rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
4568fc8068eSWojciech Macek 		    XICP_PRIORITY, &status);
45756505ec0SJustin Hibbits 		xicp_unmask(dev, irq, intr);
4588fc8068eSWojciech Macek #ifdef POWERNV
4598fc8068eSWojciech Macek 	} else {
4608fc8068eSWojciech Macek 		status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
4618fc8068eSWojciech Macek 		/* Unmask implicit for OPAL */
4628fc8068eSWojciech Macek 
4638fc8068eSWojciech Macek 		if (status != 0)
4648fc8068eSWojciech Macek 			panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
4658fc8068eSWojciech Macek 			    cpu, status);
4668fc8068eSWojciech Macek #endif
4678fc8068eSWojciech Macek 	}
4687a8d25c0SNathan Whitehorn }
4697a8d25c0SNathan Whitehorn 
4707a8d25c0SNathan Whitehorn static void
47156505ec0SJustin Hibbits xicp_eoi(device_t dev, u_int irq, void *priv)
4727a8d25c0SNathan Whitehorn {
473ef6da5e5SJustin Hibbits #ifdef POWERNV
474ef6da5e5SJustin Hibbits 	struct xicp_softc *sc;
475ef6da5e5SJustin Hibbits #endif
4767a8d25c0SNathan Whitehorn 	uint64_t xirr;
4777a8d25c0SNathan Whitehorn 
4787a8d25c0SNathan Whitehorn 	if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */
4797a8d25c0SNathan Whitehorn 		irq = XICP_IPI;
4807e524b07SJustin Hibbits 	xirr = irq | (0xff << 24);
4817a8d25c0SNathan Whitehorn 
4828fc8068eSWojciech Macek #ifdef POWERNV
483ef6da5e5SJustin Hibbits 	if (mfmsr() & PSL_HV) {
484ef6da5e5SJustin Hibbits 		sc = device_get_softc(dev);
485ef6da5e5SJustin Hibbits 		if (sc->xics_emu)
486ef6da5e5SJustin Hibbits 			opal_call(OPAL_INT_EOI, xirr);
4878fc8068eSWojciech Macek 		else
488ef6da5e5SJustin Hibbits 			bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr);
489ef6da5e5SJustin Hibbits 	} else
4908fc8068eSWojciech Macek #endif
4917a8d25c0SNathan Whitehorn 		phyp_hcall(H_EOI, xirr);
4927a8d25c0SNathan Whitehorn }
4937a8d25c0SNathan Whitehorn 
4947a8d25c0SNathan Whitehorn static void
4957a8d25c0SNathan Whitehorn xicp_ipi(device_t dev, u_int cpu)
4967a8d25c0SNathan Whitehorn {
4977a8d25c0SNathan Whitehorn 
4988fc8068eSWojciech Macek #ifdef POWERNV
499ef6da5e5SJustin Hibbits 	struct xicp_softc *sc;
500f0393bbfSWojciech Macek 	cpu = pcpu_find(cpu)->pc_hwref;
501f0393bbfSWojciech Macek 
502ef6da5e5SJustin Hibbits 	if (mfmsr() & PSL_HV) {
503ef6da5e5SJustin Hibbits 		sc = device_get_softc(dev);
504ef6da5e5SJustin Hibbits 		if (sc->xics_emu) {
505ef6da5e5SJustin Hibbits 			int64_t rv;
506ef6da5e5SJustin Hibbits 			rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY);
507ef6da5e5SJustin Hibbits 			if (rv != 0)
508ef6da5e5SJustin Hibbits 			    device_printf(dev, "IPI SET_MFRR result: %ld\n", rv);
509ef6da5e5SJustin Hibbits 		} else
5108fc8068eSWojciech Macek 			bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY);
511ef6da5e5SJustin Hibbits 	} else
5128fc8068eSWojciech Macek #endif
5137a8d25c0SNathan Whitehorn 		phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY);
5147a8d25c0SNathan Whitehorn }
5157a8d25c0SNathan Whitehorn 
5167a8d25c0SNathan Whitehorn static void
51756505ec0SJustin Hibbits xicp_mask(device_t dev, u_int irq, void *priv)
5187a8d25c0SNathan Whitehorn {
5197a8d25c0SNathan Whitehorn 	struct xicp_softc *sc = device_get_softc(dev);
5207a8d25c0SNathan Whitehorn 	cell_t status;
5217a8d25c0SNathan Whitehorn 
5227a8d25c0SNathan Whitehorn 	if (irq == MAX_XICP_IRQS)
5237a8d25c0SNathan Whitehorn 		return;
5247a8d25c0SNathan Whitehorn 
5258fc8068eSWojciech Macek 	if (rtas_exists()) {
5267a8d25c0SNathan Whitehorn 		rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status);
5278fc8068eSWojciech Macek #ifdef POWERNV
5288fc8068eSWojciech Macek 	} else {
52956505ec0SJustin Hibbits 		struct xicp_intvec *ivec = priv;
5308fc8068eSWojciech Macek 
53156505ec0SJustin Hibbits 		KASSERT(ivec != NULL, ("Masking unconfigured interrupt"));
53256505ec0SJustin Hibbits 		opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, 0xff);
5338fc8068eSWojciech Macek #endif
5348fc8068eSWojciech Macek 	}
5357a8d25c0SNathan Whitehorn }
5367a8d25c0SNathan Whitehorn 
5377a8d25c0SNathan Whitehorn static void
53856505ec0SJustin Hibbits xicp_unmask(device_t dev, u_int irq, void *priv)
5397a8d25c0SNathan Whitehorn {
5407a8d25c0SNathan Whitehorn 	struct xicp_softc *sc = device_get_softc(dev);
5417a8d25c0SNathan Whitehorn 	cell_t status;
5427a8d25c0SNathan Whitehorn 
5437a8d25c0SNathan Whitehorn 	if (irq == MAX_XICP_IRQS)
5447a8d25c0SNathan Whitehorn 		return;
5457a8d25c0SNathan Whitehorn 
5468fc8068eSWojciech Macek 	if (rtas_exists()) {
5477a8d25c0SNathan Whitehorn 		rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status);
5488fc8068eSWojciech Macek #ifdef POWERNV
5498fc8068eSWojciech Macek 	} else {
55056505ec0SJustin Hibbits 		struct xicp_intvec *ivec = priv;
5518fc8068eSWojciech Macek 
55256505ec0SJustin Hibbits 		KASSERT(ivec != NULL, ("Unmasking unconfigured interrupt"));
55356505ec0SJustin Hibbits 		opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, XICP_PRIORITY);
5548fc8068eSWojciech Macek #endif
5558fc8068eSWojciech Macek 	}
5567a8d25c0SNathan Whitehorn }
5577a8d25c0SNathan Whitehorn 
558ef6da5e5SJustin Hibbits #ifdef POWERNV
559ef6da5e5SJustin Hibbits /* This is only used on POWER9 systems with the XIVE's XICS emulation. */
560d49fc192SJustin Hibbits static void
561ef6da5e5SJustin Hibbits xicp_smp_cpu_startup(void)
562ef6da5e5SJustin Hibbits {
563ef6da5e5SJustin Hibbits 	struct xicp_softc *sc;
564ef6da5e5SJustin Hibbits 
565ef6da5e5SJustin Hibbits 	if (mfmsr() & PSL_HV) {
566ef6da5e5SJustin Hibbits 		sc = device_get_softc(root_pic);
567ef6da5e5SJustin Hibbits 
568ef6da5e5SJustin Hibbits 		if (sc->xics_emu)
569ef6da5e5SJustin Hibbits 			opal_call(OPAL_INT_SET_CPPR, 0xff);
570ef6da5e5SJustin Hibbits 	}
571ef6da5e5SJustin Hibbits }
572ef6da5e5SJustin Hibbits #endif
573