xref: /freebsd/sys/powerpc/powernv/xive.c (revision be48fe60009ed0b45f5675a26e68df2f50ca382a)
1d49fc192SJustin Hibbits /*-
2d49fc192SJustin Hibbits  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3d49fc192SJustin Hibbits  *
4d49fc192SJustin Hibbits  * Copyright 2019 Justin Hibbits
5d49fc192SJustin Hibbits  *
6d49fc192SJustin Hibbits  * Redistribution and use in source and binary forms, with or without
7d49fc192SJustin Hibbits  * modification, are permitted provided that the following conditions
8d49fc192SJustin Hibbits  * are met:
9d49fc192SJustin Hibbits  * 1. Redistributions of source code must retain the above copyright
10d49fc192SJustin Hibbits  *    notice, this list of conditions and the following disclaimer.
11d49fc192SJustin Hibbits  * 2. Redistributions in binary form must reproduce the above copyright
12d49fc192SJustin Hibbits  *    notice, this list of conditions and the following disclaimer in the
13d49fc192SJustin Hibbits  *    documentation and/or other materials provided with the distribution.
14d49fc192SJustin Hibbits  *
15d49fc192SJustin Hibbits  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16d49fc192SJustin Hibbits  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17d49fc192SJustin Hibbits  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18d49fc192SJustin Hibbits  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19d49fc192SJustin Hibbits  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20d49fc192SJustin Hibbits  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21d49fc192SJustin Hibbits  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22d49fc192SJustin Hibbits  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23d49fc192SJustin Hibbits  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24d49fc192SJustin Hibbits  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25d49fc192SJustin Hibbits  * SUCH DAMAGE.
26d49fc192SJustin Hibbits  */
27d49fc192SJustin Hibbits 
28d49fc192SJustin Hibbits #include <sys/cdefs.h>
29d49fc192SJustin Hibbits __FBSDID("$FreeBSD$");
30d49fc192SJustin Hibbits 
31d49fc192SJustin Hibbits #include "opt_platform.h"
32d49fc192SJustin Hibbits 
33d49fc192SJustin Hibbits #include <sys/param.h>
34d49fc192SJustin Hibbits #include <sys/systm.h>
35d49fc192SJustin Hibbits #include <sys/module.h>
36d49fc192SJustin Hibbits #include <sys/bus.h>
37d49fc192SJustin Hibbits #include <sys/conf.h>
38d49fc192SJustin Hibbits #include <sys/endian.h>
39d49fc192SJustin Hibbits #include <sys/kernel.h>
40e2e050c8SConrad Meyer #include <sys/lock.h>
41d49fc192SJustin Hibbits #include <sys/malloc.h>
42e2e050c8SConrad Meyer #include <sys/mutex.h>
43d49fc192SJustin Hibbits #include <sys/smp.h>
44d49fc192SJustin Hibbits 
45d49fc192SJustin Hibbits #include <vm/vm.h>
46d49fc192SJustin Hibbits #include <vm/pmap.h>
47d49fc192SJustin Hibbits 
48d49fc192SJustin Hibbits #include <machine/bus.h>
49d49fc192SJustin Hibbits #include <machine/intr_machdep.h>
50d49fc192SJustin Hibbits #include <machine/md_var.h>
51d49fc192SJustin Hibbits 
52d49fc192SJustin Hibbits #include <dev/ofw/ofw_bus.h>
53d49fc192SJustin Hibbits #include <dev/ofw/ofw_bus_subr.h>
54d49fc192SJustin Hibbits 
55d49fc192SJustin Hibbits #ifdef POWERNV
56d49fc192SJustin Hibbits #include <powerpc/powernv/opal.h>
57d49fc192SJustin Hibbits #endif
58d49fc192SJustin Hibbits 
59d49fc192SJustin Hibbits #include "pic_if.h"
60d49fc192SJustin Hibbits 
61d49fc192SJustin Hibbits #define XIVE_PRIORITY	7	/* Random non-zero number */
62d49fc192SJustin Hibbits #define MAX_XIVE_IRQS	(1<<24)	/* 24-bit XIRR field */
63d49fc192SJustin Hibbits 
64d49fc192SJustin Hibbits /* Registers */
65d49fc192SJustin Hibbits #define	XIVE_TM_QW1_OS		0x010	/* Guest OS registers */
66d49fc192SJustin Hibbits #define	XIVE_TM_QW2_HV_POOL	0x020	/* Hypervisor pool registers */
67d49fc192SJustin Hibbits #define	XIVE_TM_QW3_HV		0x030	/* Hypervisor registers */
68d49fc192SJustin Hibbits 
69d49fc192SJustin Hibbits #define	XIVE_TM_NSR	0x00
70d49fc192SJustin Hibbits #define	XIVE_TM_CPPR	0x01
71d49fc192SJustin Hibbits #define	XIVE_TM_IPB	0x02
72d49fc192SJustin Hibbits #define	XIVE_TM_LSMFB	0x03
73d49fc192SJustin Hibbits #define	XIVE_TM_ACK_CNT	0x04
74d49fc192SJustin Hibbits #define	XIVE_TM_INC	0x05
75d49fc192SJustin Hibbits #define	XIVE_TM_AGE	0x06
76d49fc192SJustin Hibbits #define	XIVE_TM_PIPR	0x07
77d49fc192SJustin Hibbits 
78d49fc192SJustin Hibbits #define	TM_WORD0	0x0
79d49fc192SJustin Hibbits #define	TM_WORD2	0x8
80d49fc192SJustin Hibbits #define	  TM_QW2W2_VP	  0x80000000
81d49fc192SJustin Hibbits 
82d49fc192SJustin Hibbits #define	XIVE_TM_SPC_ACK			0x800
83d49fc192SJustin Hibbits #define	  TM_QW3NSR_HE_SHIFT		  14
84d49fc192SJustin Hibbits #define	  TM_QW3_NSR_HE_NONE		  0
85d49fc192SJustin Hibbits #define	  TM_QW3_NSR_HE_POOL		  1
86d49fc192SJustin Hibbits #define	  TM_QW3_NSR_HE_PHYS		  2
87d49fc192SJustin Hibbits #define	  TM_QW3_NSR_HE_LSI		  3
88d49fc192SJustin Hibbits #define	XIVE_TM_SPC_PULL_POOL_CTX	0x828
89d49fc192SJustin Hibbits 
90d49fc192SJustin Hibbits #define	XIVE_IRQ_LOAD_EOI	0x000
91d49fc192SJustin Hibbits #define	XIVE_IRQ_STORE_EOI	0x400
92d49fc192SJustin Hibbits #define	XIVE_IRQ_PQ_00		0xc00
93d49fc192SJustin Hibbits #define	XIVE_IRQ_PQ_01		0xd00
94d49fc192SJustin Hibbits 
95d49fc192SJustin Hibbits #define	XIVE_IRQ_VAL_P		0x02
96d49fc192SJustin Hibbits #define	XIVE_IRQ_VAL_Q		0x01
97d49fc192SJustin Hibbits 
98d49fc192SJustin Hibbits struct xive_softc;
99d49fc192SJustin Hibbits struct xive_irq;
100d49fc192SJustin Hibbits 
101d49fc192SJustin Hibbits extern void (*powernv_smp_ap_extra_init)(void);
102d49fc192SJustin Hibbits 
103d49fc192SJustin Hibbits /* Private support */
104d49fc192SJustin Hibbits static void	xive_setup_cpu(void);
105d49fc192SJustin Hibbits static void	xive_smp_cpu_startup(void);
106d49fc192SJustin Hibbits static void	xive_init_irq(struct xive_irq *irqd, u_int irq);
107d49fc192SJustin Hibbits static struct xive_irq	*xive_configure_irq(u_int irq);
108d49fc192SJustin Hibbits static int	xive_provision_page(struct xive_softc *sc);
109d49fc192SJustin Hibbits 
110d49fc192SJustin Hibbits /* Interfaces */
111d49fc192SJustin Hibbits static int	xive_probe(device_t);
112d49fc192SJustin Hibbits static int	xive_attach(device_t);
113d49fc192SJustin Hibbits static int	xics_probe(device_t);
114d49fc192SJustin Hibbits static int	xics_attach(device_t);
115d49fc192SJustin Hibbits 
116d49fc192SJustin Hibbits static void	xive_bind(device_t, u_int, cpuset_t, void **);
117d49fc192SJustin Hibbits static void	xive_dispatch(device_t, struct trapframe *);
118d49fc192SJustin Hibbits static void	xive_enable(device_t, u_int, u_int, void **);
119d49fc192SJustin Hibbits static void	xive_eoi(device_t, u_int, void *);
120d49fc192SJustin Hibbits static void	xive_ipi(device_t, u_int);
121d49fc192SJustin Hibbits static void	xive_mask(device_t, u_int, void *);
122d49fc192SJustin Hibbits static void	xive_unmask(device_t, u_int, void *);
123d49fc192SJustin Hibbits static void	xive_translate_code(device_t dev, u_int irq, int code,
124d49fc192SJustin Hibbits 		    enum intr_trigger *trig, enum intr_polarity *pol);
125d49fc192SJustin Hibbits 
126d49fc192SJustin Hibbits static device_method_t  xive_methods[] = {
127d49fc192SJustin Hibbits 	/* Device interface */
128d49fc192SJustin Hibbits 	DEVMETHOD(device_probe,		xive_probe),
129d49fc192SJustin Hibbits 	DEVMETHOD(device_attach,	xive_attach),
130d49fc192SJustin Hibbits 
131d49fc192SJustin Hibbits 	/* PIC interface */
132d49fc192SJustin Hibbits 	DEVMETHOD(pic_bind,		xive_bind),
133d49fc192SJustin Hibbits 	DEVMETHOD(pic_dispatch,		xive_dispatch),
134d49fc192SJustin Hibbits 	DEVMETHOD(pic_enable,		xive_enable),
135d49fc192SJustin Hibbits 	DEVMETHOD(pic_eoi,		xive_eoi),
136d49fc192SJustin Hibbits 	DEVMETHOD(pic_ipi,		xive_ipi),
137d49fc192SJustin Hibbits 	DEVMETHOD(pic_mask,		xive_mask),
138d49fc192SJustin Hibbits 	DEVMETHOD(pic_unmask,		xive_unmask),
139d49fc192SJustin Hibbits 	DEVMETHOD(pic_translate_code,	xive_translate_code),
140d49fc192SJustin Hibbits 
141d49fc192SJustin Hibbits 	DEVMETHOD_END
142d49fc192SJustin Hibbits };
143d49fc192SJustin Hibbits 
144d49fc192SJustin Hibbits static device_method_t  xics_methods[] = {
145d49fc192SJustin Hibbits 	/* Device interface */
146d49fc192SJustin Hibbits 	DEVMETHOD(device_probe,		xics_probe),
147d49fc192SJustin Hibbits 	DEVMETHOD(device_attach,	xics_attach),
148d49fc192SJustin Hibbits 
149d49fc192SJustin Hibbits 	DEVMETHOD_END
150d49fc192SJustin Hibbits };
151d49fc192SJustin Hibbits 
152d49fc192SJustin Hibbits struct xive_softc {
153d49fc192SJustin Hibbits 	struct mtx sc_mtx;
154d49fc192SJustin Hibbits 	struct resource *sc_mem;
155d49fc192SJustin Hibbits 	vm_size_t	sc_prov_page_size;
156d49fc192SJustin Hibbits 	uint32_t	sc_offset;
157d49fc192SJustin Hibbits };
158d49fc192SJustin Hibbits 
159d49fc192SJustin Hibbits struct xive_queue {
160d49fc192SJustin Hibbits 	uint32_t	*q_page;
161d49fc192SJustin Hibbits 	uint32_t	*q_eoi_page;
162d49fc192SJustin Hibbits 	uint32_t	 q_toggle;
163d49fc192SJustin Hibbits 	uint32_t	 q_size;
164d49fc192SJustin Hibbits 	uint32_t	 q_index;
165d49fc192SJustin Hibbits 	uint32_t	 q_mask;
166d49fc192SJustin Hibbits };
167d49fc192SJustin Hibbits 
168d49fc192SJustin Hibbits struct xive_irq {
169d49fc192SJustin Hibbits 	uint32_t	girq;
170d49fc192SJustin Hibbits 	uint32_t	lirq;
171d49fc192SJustin Hibbits 	uint64_t	vp;
172d49fc192SJustin Hibbits 	uint64_t	flags;
173d49fc192SJustin Hibbits #define	OPAL_XIVE_IRQ_SHIFT_BUG		0x00000008
174d49fc192SJustin Hibbits #define	OPAL_XIVE_IRQ_LSI		0x00000004
175d49fc192SJustin Hibbits #define	OPAL_XIVE_IRQ_STORE_EOI		0x00000002
176d49fc192SJustin Hibbits #define	OPAL_XIVE_IRQ_TRIGGER_PAGE	0x00000001
177d49fc192SJustin Hibbits 	uint8_t	prio;
178d49fc192SJustin Hibbits 	vm_offset_t	eoi_page;
179d49fc192SJustin Hibbits 	vm_offset_t	trig_page;
180d49fc192SJustin Hibbits 	vm_size_t	esb_size;
181d49fc192SJustin Hibbits 	int		chip;
182d49fc192SJustin Hibbits };
183d49fc192SJustin Hibbits 
184d49fc192SJustin Hibbits struct xive_cpu {
185d49fc192SJustin Hibbits 	uint64_t	vp;
186d49fc192SJustin Hibbits 	uint64_t	flags;
187d49fc192SJustin Hibbits 	struct xive_irq	ipi_data;
188d49fc192SJustin Hibbits 	struct xive_queue	queue; /* We only use a single queue for now. */
189d49fc192SJustin Hibbits 	uint64_t	cam;
190d49fc192SJustin Hibbits 	uint32_t	chip;
191d49fc192SJustin Hibbits };
192d49fc192SJustin Hibbits 
193d49fc192SJustin Hibbits static driver_t xive_driver = {
194d49fc192SJustin Hibbits 	"xive",
195d49fc192SJustin Hibbits 	xive_methods,
196d49fc192SJustin Hibbits 	sizeof(struct xive_softc)
197d49fc192SJustin Hibbits };
198d49fc192SJustin Hibbits 
199d49fc192SJustin Hibbits static driver_t xics_driver = {
200d49fc192SJustin Hibbits 	"xivevc",
201d49fc192SJustin Hibbits 	xics_methods,
202d49fc192SJustin Hibbits 	0
203d49fc192SJustin Hibbits };
204d49fc192SJustin Hibbits 
205d49fc192SJustin Hibbits static devclass_t xive_devclass;
206d49fc192SJustin Hibbits static devclass_t xics_devclass;
207d49fc192SJustin Hibbits 
208d49fc192SJustin Hibbits EARLY_DRIVER_MODULE(xive, ofwbus, xive_driver, xive_devclass, 0, 0,
209d49fc192SJustin Hibbits     BUS_PASS_INTERRUPT-1);
210d49fc192SJustin Hibbits EARLY_DRIVER_MODULE(xivevc, ofwbus, xics_driver, xics_devclass, 0, 0,
211d49fc192SJustin Hibbits     BUS_PASS_INTERRUPT);
212d49fc192SJustin Hibbits 
213d49fc192SJustin Hibbits MALLOC_DEFINE(M_XIVE, "xive", "XIVE Memory");
214d49fc192SJustin Hibbits 
215d49fc192SJustin Hibbits DPCPU_DEFINE_STATIC(struct xive_cpu, xive_cpu_data);
216d49fc192SJustin Hibbits 
217d49fc192SJustin Hibbits static int xive_ipi_vector = -1;
218d49fc192SJustin Hibbits 
219d49fc192SJustin Hibbits /*
220d49fc192SJustin Hibbits  * XIVE Exploitation mode driver.
221d49fc192SJustin Hibbits  *
222d49fc192SJustin Hibbits  * The XIVE, present in the POWER9 CPU, can run in two modes: XICS emulation
223d49fc192SJustin Hibbits  * mode, and "Exploitation mode".  XICS emulation mode is compatible with the
224d49fc192SJustin Hibbits  * POWER8 and earlier XICS interrupt controller, using OPAL calls to emulate
225d49fc192SJustin Hibbits  * hypervisor calls and memory accesses.  Exploitation mode gives us raw access
226d49fc192SJustin Hibbits  * to the XIVE MMIO, improving performance significantly.
227d49fc192SJustin Hibbits  *
228d49fc192SJustin Hibbits  * The XIVE controller is a very bizarre interrupt controller.  It uses queues
229d49fc192SJustin Hibbits  * in memory to pass interrupts around, and maps itself into 512GB of physical
230d49fc192SJustin Hibbits  * device address space, giving each interrupt in the system one or more pages
231d49fc192SJustin Hibbits  * of address space.  An IRQ is tied to a virtual processor, which could be a
232d49fc192SJustin Hibbits  * physical CPU thread, or a guest CPU thread (LPAR running on a physical
233d49fc192SJustin Hibbits  * thread).  Thus, the controller can route interrupts directly to guest OSes
234d49fc192SJustin Hibbits  * bypassing processing by the hypervisor, thereby improving performance of the
235d49fc192SJustin Hibbits  * guest OS.
236d49fc192SJustin Hibbits  *
237d49fc192SJustin Hibbits  * An IRQ, in addition to being tied to a virtual processor, has one or two
238d49fc192SJustin Hibbits  * page mappings: an EOI page, and an optional trigger page.  The trigger page
239d49fc192SJustin Hibbits  * could be the same as the EOI page.  Level-sensitive interrupts (LSIs) don't
240d49fc192SJustin Hibbits  * have a trigger page, as they're external interrupts controlled by physical
241d49fc192SJustin Hibbits  * lines.  MSIs and IPIs have trigger pages.  An IPI is really just another IRQ
242d49fc192SJustin Hibbits  * in the XIVE, which is triggered by software.
243d49fc192SJustin Hibbits  *
244d49fc192SJustin Hibbits  * An interesting behavior of the XIVE controller is that oftentimes the
245d49fc192SJustin Hibbits  * contents of an address location don't actually matter, but the direction of
246d49fc192SJustin Hibbits  * the action is the signifier (read vs write), and the address is significant.
247d49fc192SJustin Hibbits  * Hence, masking and unmasking an interrupt is done by reading different
248d49fc192SJustin Hibbits  * addresses in the EOI page, and triggering an interrupt consists of writing to
249d49fc192SJustin Hibbits  * the trigger page.
250d49fc192SJustin Hibbits  *
251d49fc192SJustin Hibbits  * Additionally, the MMIO region mapped is CPU-sensitive, just like the
252d49fc192SJustin Hibbits  * per-processor register space (private access) in OpenPIC.  In order for a CPU
253d49fc192SJustin Hibbits  * to receive interrupts it must itself configure its CPPR (Current Processor
254d49fc192SJustin Hibbits  * Priority Register), it cannot be set by any other processor.  This
255d49fc192SJustin Hibbits  * necessitates the xive_smp_cpu_startup() function.
256d49fc192SJustin Hibbits  *
257d49fc192SJustin Hibbits  * Queues are pages of memory, sized powers-of-two, that are shared with the
258d49fc192SJustin Hibbits  * XIVE.  The XIVE writes into the queue with an alternating polarity bit, which
259d49fc192SJustin Hibbits  * flips when the queue wraps.
260d49fc192SJustin Hibbits  */
261d49fc192SJustin Hibbits 
262d49fc192SJustin Hibbits /*
263d49fc192SJustin Hibbits  * Offset-based read/write interfaces.
264d49fc192SJustin Hibbits  */
265d49fc192SJustin Hibbits static uint16_t
266d49fc192SJustin Hibbits xive_read_2(struct xive_softc *sc, bus_size_t offset)
267d49fc192SJustin Hibbits {
268d49fc192SJustin Hibbits 
269d49fc192SJustin Hibbits 	return (bus_read_2(sc->sc_mem, sc->sc_offset + offset));
270d49fc192SJustin Hibbits }
271d49fc192SJustin Hibbits 
272d49fc192SJustin Hibbits static void
273d49fc192SJustin Hibbits xive_write_1(struct xive_softc *sc, bus_size_t offset, uint8_t val)
274d49fc192SJustin Hibbits {
275d49fc192SJustin Hibbits 
276d49fc192SJustin Hibbits 	bus_write_1(sc->sc_mem, sc->sc_offset + offset, val);
277d49fc192SJustin Hibbits }
278d49fc192SJustin Hibbits 
279d49fc192SJustin Hibbits /* EOI and Trigger page access interfaces. */
280d49fc192SJustin Hibbits static uint64_t
281d49fc192SJustin Hibbits xive_read_mmap8(vm_offset_t addr)
282d49fc192SJustin Hibbits {
283d49fc192SJustin Hibbits 	return (*(volatile uint64_t *)addr);
284d49fc192SJustin Hibbits }
285d49fc192SJustin Hibbits 
286d49fc192SJustin Hibbits static void
287d49fc192SJustin Hibbits xive_write_mmap8(vm_offset_t addr, uint64_t val)
288d49fc192SJustin Hibbits {
289d49fc192SJustin Hibbits 	*(uint64_t *)(addr) = val;
290d49fc192SJustin Hibbits }
291d49fc192SJustin Hibbits 
292d49fc192SJustin Hibbits /* Device interfaces. */
293d49fc192SJustin Hibbits static int
294d49fc192SJustin Hibbits xive_probe(device_t dev)
295d49fc192SJustin Hibbits {
296d49fc192SJustin Hibbits 
297d49fc192SJustin Hibbits 	if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-pe"))
298d49fc192SJustin Hibbits 		return (ENXIO);
299d49fc192SJustin Hibbits 
300d49fc192SJustin Hibbits 	device_set_desc(dev, "External Interrupt Virtualization Engine");
301d49fc192SJustin Hibbits 
302d49fc192SJustin Hibbits 	/* Make sure we always win against the xicp driver. */
303d49fc192SJustin Hibbits 	return (BUS_PROBE_DEFAULT);
304d49fc192SJustin Hibbits }
305d49fc192SJustin Hibbits 
306d49fc192SJustin Hibbits static int
307d49fc192SJustin Hibbits xics_probe(device_t dev)
308d49fc192SJustin Hibbits {
309d49fc192SJustin Hibbits 
310d49fc192SJustin Hibbits 	if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-vc"))
311d49fc192SJustin Hibbits 		return (ENXIO);
312d49fc192SJustin Hibbits 
313d49fc192SJustin Hibbits 	device_set_desc(dev, "External Interrupt Virtualization Engine Root");
314d49fc192SJustin Hibbits 	return (BUS_PROBE_DEFAULT);
315d49fc192SJustin Hibbits }
316d49fc192SJustin Hibbits 
317d49fc192SJustin Hibbits static int
318d49fc192SJustin Hibbits xive_attach(device_t dev)
319d49fc192SJustin Hibbits {
320d49fc192SJustin Hibbits 	struct xive_softc *sc = device_get_softc(dev);
321d49fc192SJustin Hibbits 	struct xive_cpu *xive_cpud;
322d49fc192SJustin Hibbits 	phandle_t phandle = ofw_bus_get_node(dev);
323d49fc192SJustin Hibbits 	int64_t vp_block;
324d49fc192SJustin Hibbits 	int error;
325d49fc192SJustin Hibbits 	int rid;
326d49fc192SJustin Hibbits 	int i, order;
327d49fc192SJustin Hibbits 	uint64_t vp_id;
328d49fc192SJustin Hibbits 	int64_t ipi_irq;
329d49fc192SJustin Hibbits 
330d49fc192SJustin Hibbits 	opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EXP);
331d49fc192SJustin Hibbits 
332d49fc192SJustin Hibbits 	error = OF_getencprop(phandle, "ibm,xive-provision-page-size",
333d49fc192SJustin Hibbits 	    (pcell_t *)&sc->sc_prov_page_size, sizeof(sc->sc_prov_page_size));
334d49fc192SJustin Hibbits 
335d49fc192SJustin Hibbits 	rid = 1;	/* Get the Hypervisor-level register set. */
336d49fc192SJustin Hibbits 	sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
337d49fc192SJustin Hibbits 	    &rid, RF_ACTIVE);
338d49fc192SJustin Hibbits 	sc->sc_offset = XIVE_TM_QW3_HV;
339d49fc192SJustin Hibbits 
340d49fc192SJustin Hibbits 	mtx_init(&sc->sc_mtx, "XIVE", NULL, MTX_DEF);
341d49fc192SJustin Hibbits 
34269576451SBrandon Bergren 	/* Workaround for qemu single-thread powernv */
34369576451SBrandon Bergren 	if (mp_maxid == 0)
34469576451SBrandon Bergren 		order = 1;
34569576451SBrandon Bergren 	else
346d49fc192SJustin Hibbits 		order = fls(mp_maxid + (mp_maxid - 1)) - 1;
347d49fc192SJustin Hibbits 
348d49fc192SJustin Hibbits 	do {
349d49fc192SJustin Hibbits 		vp_block = opal_call(OPAL_XIVE_ALLOCATE_VP_BLOCK, order);
350d49fc192SJustin Hibbits 		if (vp_block == OPAL_BUSY)
351d49fc192SJustin Hibbits 			DELAY(10);
352d49fc192SJustin Hibbits 		else if (vp_block == OPAL_XIVE_PROVISIONING)
353d49fc192SJustin Hibbits 			xive_provision_page(sc);
354d49fc192SJustin Hibbits 		else
355d49fc192SJustin Hibbits 			break;
356d49fc192SJustin Hibbits 	} while (1);
357d49fc192SJustin Hibbits 
358d49fc192SJustin Hibbits 	if (vp_block < 0) {
359d49fc192SJustin Hibbits 		device_printf(dev,
360d49fc192SJustin Hibbits 		    "Unable to allocate VP block.  Opal error %d\n",
361d49fc192SJustin Hibbits 		    (int)vp_block);
362d49fc192SJustin Hibbits 		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
363d49fc192SJustin Hibbits 		return (ENXIO);
364d49fc192SJustin Hibbits 	}
365d49fc192SJustin Hibbits 
366d49fc192SJustin Hibbits 	/*
367d49fc192SJustin Hibbits 	 * Set up the VPs.  Try to do as much as we can in attach, to lessen
368d49fc192SJustin Hibbits 	 * what's needed at AP spawn time.
369d49fc192SJustin Hibbits 	 */
370d49fc192SJustin Hibbits 	CPU_FOREACH(i) {
371d49fc192SJustin Hibbits 		vp_id = pcpu_find(i)->pc_hwref;
372d49fc192SJustin Hibbits 
373d49fc192SJustin Hibbits 		xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data);
374d49fc192SJustin Hibbits 		xive_cpud->vp = vp_id + vp_block;
375d49fc192SJustin Hibbits 		opal_call(OPAL_XIVE_GET_VP_INFO, xive_cpud->vp, NULL,
376d49fc192SJustin Hibbits 		    vtophys(&xive_cpud->cam), NULL, vtophys(&xive_cpud->chip));
377d49fc192SJustin Hibbits 
378f9acb7a8SBrandon Bergren 		xive_cpud->cam = be64toh(xive_cpud->cam);
379f9acb7a8SBrandon Bergren 		xive_cpud->chip = be64toh(xive_cpud->chip);
380f9acb7a8SBrandon Bergren 
381d49fc192SJustin Hibbits 		/* Allocate the queue page and populate the queue state data. */
382d49fc192SJustin Hibbits 		xive_cpud->queue.q_page = contigmalloc(PAGE_SIZE, M_XIVE,
383d49fc192SJustin Hibbits 		    M_ZERO | M_WAITOK, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
384d49fc192SJustin Hibbits 		xive_cpud->queue.q_size = 1 << PAGE_SHIFT;
385d49fc192SJustin Hibbits 		xive_cpud->queue.q_mask =
386d49fc192SJustin Hibbits 		    ((xive_cpud->queue.q_size / sizeof(int)) - 1);
387d49fc192SJustin Hibbits 		xive_cpud->queue.q_toggle = 0;
388d49fc192SJustin Hibbits 		xive_cpud->queue.q_index = 0;
389d49fc192SJustin Hibbits 		do {
390d49fc192SJustin Hibbits 			error = opal_call(OPAL_XIVE_SET_VP_INFO, xive_cpud->vp,
391d49fc192SJustin Hibbits 			    OPAL_XIVE_VP_ENABLED, 0);
392d49fc192SJustin Hibbits 		} while (error == OPAL_BUSY);
393d49fc192SJustin Hibbits 		error = opal_call(OPAL_XIVE_SET_QUEUE_INFO, vp_id,
394d49fc192SJustin Hibbits 		    XIVE_PRIORITY, vtophys(xive_cpud->queue.q_page), PAGE_SHIFT,
395d49fc192SJustin Hibbits 		    OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED);
396d49fc192SJustin Hibbits 
397d49fc192SJustin Hibbits 		do {
398d49fc192SJustin Hibbits 			ipi_irq = opal_call(OPAL_XIVE_ALLOCATE_IRQ,
399d49fc192SJustin Hibbits 			    xive_cpud->chip);
400d49fc192SJustin Hibbits 		} while (ipi_irq == OPAL_BUSY);
401d49fc192SJustin Hibbits 
402d49fc192SJustin Hibbits 		if (ipi_irq < 0)
403d49fc192SJustin Hibbits 			device_printf(root_pic,
404d49fc192SJustin Hibbits 			    "Failed allocating IPI.  OPAL error %d\n",
405d49fc192SJustin Hibbits 			    (int)ipi_irq);
406d49fc192SJustin Hibbits 		else {
407d49fc192SJustin Hibbits 			xive_init_irq(&xive_cpud->ipi_data, ipi_irq);
408d49fc192SJustin Hibbits 			xive_cpud->ipi_data.vp = vp_id;
409d49fc192SJustin Hibbits 			xive_cpud->ipi_data.lirq = MAX_XIVE_IRQS;
410d49fc192SJustin Hibbits 			opal_call(OPAL_XIVE_SET_IRQ_CONFIG, ipi_irq,
411d49fc192SJustin Hibbits 			    xive_cpud->ipi_data.vp, XIVE_PRIORITY,
412d49fc192SJustin Hibbits 			    MAX_XIVE_IRQS);
413d49fc192SJustin Hibbits 		}
414d49fc192SJustin Hibbits 	}
415d49fc192SJustin Hibbits 
416d49fc192SJustin Hibbits 	powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XIVE_IRQS,
417d49fc192SJustin Hibbits 	    1 /* Number of IPIs */, FALSE);
418d49fc192SJustin Hibbits 	root_pic = dev;
419d49fc192SJustin Hibbits 
420d49fc192SJustin Hibbits 	xive_setup_cpu();
421d49fc192SJustin Hibbits 	powernv_smp_ap_extra_init = xive_smp_cpu_startup;
422d49fc192SJustin Hibbits 
423d49fc192SJustin Hibbits 	return (0);
424d49fc192SJustin Hibbits }
425d49fc192SJustin Hibbits 
426d49fc192SJustin Hibbits static int
427d49fc192SJustin Hibbits xics_attach(device_t dev)
428d49fc192SJustin Hibbits {
429d49fc192SJustin Hibbits 	phandle_t phandle = ofw_bus_get_node(dev);
430d49fc192SJustin Hibbits 
431d49fc192SJustin Hibbits 	/* The XIVE (root PIC) will handle all our interrupts */
432d49fc192SJustin Hibbits 	powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
433d49fc192SJustin Hibbits 	    MAX_XIVE_IRQS, 1 /* Number of IPIs */, FALSE);
434d49fc192SJustin Hibbits 
435d49fc192SJustin Hibbits 	return (0);
436d49fc192SJustin Hibbits }
437d49fc192SJustin Hibbits 
438d49fc192SJustin Hibbits /*
439d49fc192SJustin Hibbits  * PIC I/F methods.
440d49fc192SJustin Hibbits  */
441d49fc192SJustin Hibbits 
442d49fc192SJustin Hibbits static void
443d49fc192SJustin Hibbits xive_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
444d49fc192SJustin Hibbits {
445d49fc192SJustin Hibbits 	struct xive_irq *irqd;
446d49fc192SJustin Hibbits 	int cpu;
447d49fc192SJustin Hibbits 	int ncpus, i, error;
448d49fc192SJustin Hibbits 
449d49fc192SJustin Hibbits 	if (*priv == NULL)
450d49fc192SJustin Hibbits 		*priv = xive_configure_irq(irq);
451d49fc192SJustin Hibbits 
452d49fc192SJustin Hibbits 	irqd = *priv;
453d49fc192SJustin Hibbits 
454d49fc192SJustin Hibbits 	/*
455d49fc192SJustin Hibbits 	 * This doesn't appear to actually support affinity groups, so pick a
456d49fc192SJustin Hibbits 	 * random CPU.
457d49fc192SJustin Hibbits 	 */
458d49fc192SJustin Hibbits 	ncpus = 0;
459d49fc192SJustin Hibbits 	CPU_FOREACH(cpu)
460d49fc192SJustin Hibbits 		if (CPU_ISSET(cpu, &cpumask)) ncpus++;
461d49fc192SJustin Hibbits 
462d49fc192SJustin Hibbits 	i = mftb() % ncpus;
463d49fc192SJustin Hibbits 	ncpus = 0;
464d49fc192SJustin Hibbits 	CPU_FOREACH(cpu) {
465d49fc192SJustin Hibbits 		if (!CPU_ISSET(cpu, &cpumask))
466d49fc192SJustin Hibbits 			continue;
467d49fc192SJustin Hibbits 		if (ncpus == i)
468d49fc192SJustin Hibbits 			break;
469d49fc192SJustin Hibbits 		ncpus++;
470d49fc192SJustin Hibbits 	}
471d49fc192SJustin Hibbits 
472b94b2fcdSBrandon Bergren 	opal_call(OPAL_XIVE_SYNC, OPAL_XIVE_SYNC_QUEUE, irq);
473d49fc192SJustin Hibbits 
474d49fc192SJustin Hibbits 	irqd->vp = pcpu_find(cpu)->pc_hwref;
475d49fc192SJustin Hibbits 	error = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, irqd->vp,
476d49fc192SJustin Hibbits 	    XIVE_PRIORITY, irqd->lirq);
477d49fc192SJustin Hibbits 
478d49fc192SJustin Hibbits 	if (error < 0)
479d49fc192SJustin Hibbits 		panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
480d49fc192SJustin Hibbits 
481d49fc192SJustin Hibbits 	xive_eoi(dev, irq, irqd);
482d49fc192SJustin Hibbits }
483d49fc192SJustin Hibbits 
484d49fc192SJustin Hibbits /* Read the next entry in the queue page and update the index. */
485d49fc192SJustin Hibbits static int
486d49fc192SJustin Hibbits xive_read_eq(struct xive_queue *q)
487d49fc192SJustin Hibbits {
488d49fc192SJustin Hibbits 	uint32_t i = be32toh(q->q_page[q->q_index]);
489d49fc192SJustin Hibbits 
490d49fc192SJustin Hibbits 	/* Check validity, using current queue polarity. */
491d49fc192SJustin Hibbits 	if ((i >> 31) == q->q_toggle)
492d49fc192SJustin Hibbits 		return (0);
493d49fc192SJustin Hibbits 
494d49fc192SJustin Hibbits 	q->q_index = (q->q_index + 1) & q->q_mask;
495d49fc192SJustin Hibbits 
496d49fc192SJustin Hibbits 	if (q->q_index == 0)
497d49fc192SJustin Hibbits 		q->q_toggle ^= 1;
498d49fc192SJustin Hibbits 
499d49fc192SJustin Hibbits 	return (i & 0x7fffffff);
500d49fc192SJustin Hibbits }
501d49fc192SJustin Hibbits 
502d49fc192SJustin Hibbits static void
503d49fc192SJustin Hibbits xive_dispatch(device_t dev, struct trapframe *tf)
504d49fc192SJustin Hibbits {
505d49fc192SJustin Hibbits 	struct xive_softc *sc;
506d49fc192SJustin Hibbits 	struct xive_cpu *xive_cpud;
507d49fc192SJustin Hibbits 	uint32_t vector;
508d49fc192SJustin Hibbits 	uint16_t ack;
509d49fc192SJustin Hibbits 	uint8_t cppr, he;
510d49fc192SJustin Hibbits 
511d49fc192SJustin Hibbits 	sc = device_get_softc(dev);
512d49fc192SJustin Hibbits 
51360185d89SBrandon Bergren 	xive_cpud = DPCPU_PTR(xive_cpu_data);
514d49fc192SJustin Hibbits 	for (;;) {
515d49fc192SJustin Hibbits 		ack = xive_read_2(sc, XIVE_TM_SPC_ACK);
516d49fc192SJustin Hibbits 		cppr = (ack & 0xff);
517d49fc192SJustin Hibbits 
518d49fc192SJustin Hibbits 		he = ack >> TM_QW3NSR_HE_SHIFT;
519d49fc192SJustin Hibbits 
520d49fc192SJustin Hibbits 		if (he == TM_QW3_NSR_HE_NONE)
521d49fc192SJustin Hibbits 			break;
52260185d89SBrandon Bergren 
52360185d89SBrandon Bergren 		else if (__predict_false(he != TM_QW3_NSR_HE_PHYS)) {
52460185d89SBrandon Bergren 			/*
52560185d89SBrandon Bergren 			 * We don't support TM_QW3_NSR_HE_POOL or
52660185d89SBrandon Bergren 			 * TM_QW3_NSR_HE_LSI interrupts.
52760185d89SBrandon Bergren 			 */
528d49fc192SJustin Hibbits 			device_printf(dev,
529d49fc192SJustin Hibbits 			    "Unexpected interrupt he type: %d\n", he);
530d49fc192SJustin Hibbits 			goto end;
531d49fc192SJustin Hibbits 		}
532d49fc192SJustin Hibbits 
533d49fc192SJustin Hibbits 		xive_write_1(sc, XIVE_TM_CPPR, cppr);
534d49fc192SJustin Hibbits 
535d49fc192SJustin Hibbits 		for (;;) {
536d49fc192SJustin Hibbits 			vector = xive_read_eq(&xive_cpud->queue);
537d49fc192SJustin Hibbits 
538d49fc192SJustin Hibbits 			if (vector == 0)
539d49fc192SJustin Hibbits 				break;
540d49fc192SJustin Hibbits 
541d49fc192SJustin Hibbits 			if (vector == MAX_XIVE_IRQS)
542d49fc192SJustin Hibbits 				vector = xive_ipi_vector;
543d49fc192SJustin Hibbits 
544d49fc192SJustin Hibbits 			powerpc_dispatch_intr(vector, tf);
545d49fc192SJustin Hibbits 		}
546d49fc192SJustin Hibbits 	}
547d49fc192SJustin Hibbits end:
548d49fc192SJustin Hibbits 	xive_write_1(sc, XIVE_TM_CPPR, 0xff);
549d49fc192SJustin Hibbits }
550d49fc192SJustin Hibbits 
551d49fc192SJustin Hibbits static void
552d49fc192SJustin Hibbits xive_enable(device_t dev, u_int irq, u_int vector, void **priv)
553d49fc192SJustin Hibbits {
554d49fc192SJustin Hibbits 	struct xive_irq *irqd;
555d49fc192SJustin Hibbits 	cell_t status, cpu;
556d49fc192SJustin Hibbits 
557d49fc192SJustin Hibbits 	if (irq == MAX_XIVE_IRQS) {
558d49fc192SJustin Hibbits 		if (xive_ipi_vector == -1)
559d49fc192SJustin Hibbits 			xive_ipi_vector = vector;
560d49fc192SJustin Hibbits 		return;
561d49fc192SJustin Hibbits 	}
562d49fc192SJustin Hibbits 	if (*priv == NULL)
563d49fc192SJustin Hibbits 		*priv = xive_configure_irq(irq);
564d49fc192SJustin Hibbits 
565d49fc192SJustin Hibbits 	irqd = *priv;
566d49fc192SJustin Hibbits 
567d49fc192SJustin Hibbits 	/* Bind to this CPU to start */
568d49fc192SJustin Hibbits 	cpu = PCPU_GET(hwref);
569d49fc192SJustin Hibbits 	irqd->lirq = vector;
570d49fc192SJustin Hibbits 
571d49fc192SJustin Hibbits 	for (;;) {
572d49fc192SJustin Hibbits 		status = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, cpu,
573d49fc192SJustin Hibbits 		    XIVE_PRIORITY, vector);
574d49fc192SJustin Hibbits 		if (status != OPAL_BUSY)
575d49fc192SJustin Hibbits 			break;
576d49fc192SJustin Hibbits 		DELAY(10);
577d49fc192SJustin Hibbits 	}
578d49fc192SJustin Hibbits 
579d49fc192SJustin Hibbits 	if (status != 0)
580d49fc192SJustin Hibbits 		panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
581d49fc192SJustin Hibbits 		    cpu, status);
582d49fc192SJustin Hibbits 
583d49fc192SJustin Hibbits 	xive_unmask(dev, irq, *priv);
584d49fc192SJustin Hibbits }
585d49fc192SJustin Hibbits 
586d49fc192SJustin Hibbits static void
587d49fc192SJustin Hibbits xive_eoi(device_t dev, u_int irq, void *priv)
588d49fc192SJustin Hibbits {
589d49fc192SJustin Hibbits 	struct xive_irq *rirq;
590d49fc192SJustin Hibbits 	struct xive_cpu *cpud;
591d49fc192SJustin Hibbits 	uint8_t eoi_val;
592d49fc192SJustin Hibbits 
593d49fc192SJustin Hibbits 	if (irq == MAX_XIVE_IRQS) {
594d49fc192SJustin Hibbits 		cpud = DPCPU_PTR(xive_cpu_data);
595d49fc192SJustin Hibbits 		rirq = &cpud->ipi_data;
596d49fc192SJustin Hibbits 	} else
597d49fc192SJustin Hibbits 		rirq = priv;
598d49fc192SJustin Hibbits 
599*be48fe60SJustin Hibbits 	if (rirq->flags & OPAL_XIVE_IRQ_STORE_EOI)
600d49fc192SJustin Hibbits 		xive_write_mmap8(rirq->eoi_page + XIVE_IRQ_STORE_EOI, 0);
601d49fc192SJustin Hibbits 	else if (rirq->flags & OPAL_XIVE_IRQ_LSI)
602d49fc192SJustin Hibbits 		xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_LOAD_EOI);
603d49fc192SJustin Hibbits 	else {
604d49fc192SJustin Hibbits 		eoi_val = xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
605d49fc192SJustin Hibbits 		if ((eoi_val & XIVE_IRQ_VAL_Q) && rirq->trig_page != 0)
606d49fc192SJustin Hibbits 			xive_write_mmap8(rirq->trig_page, 0);
607d49fc192SJustin Hibbits 	}
608d49fc192SJustin Hibbits }
609d49fc192SJustin Hibbits 
610d49fc192SJustin Hibbits static void
611d49fc192SJustin Hibbits xive_ipi(device_t dev, u_int cpu)
612d49fc192SJustin Hibbits {
613d49fc192SJustin Hibbits 	struct xive_cpu *xive_cpud;
614d49fc192SJustin Hibbits 
615d49fc192SJustin Hibbits 	xive_cpud = DPCPU_ID_PTR(cpu, xive_cpu_data);
616d49fc192SJustin Hibbits 
617d49fc192SJustin Hibbits 	if (xive_cpud->ipi_data.trig_page == 0)
618d49fc192SJustin Hibbits 		return;
619d49fc192SJustin Hibbits 	xive_write_mmap8(xive_cpud->ipi_data.trig_page, 0);
620d49fc192SJustin Hibbits }
621d49fc192SJustin Hibbits 
622d49fc192SJustin Hibbits static void
623d49fc192SJustin Hibbits xive_mask(device_t dev, u_int irq, void *priv)
624d49fc192SJustin Hibbits {
625d49fc192SJustin Hibbits 	struct xive_irq *rirq;
626d49fc192SJustin Hibbits 
627d49fc192SJustin Hibbits 	/* Never mask IPIs */
628d49fc192SJustin Hibbits 	if (irq == MAX_XIVE_IRQS)
629d49fc192SJustin Hibbits 		return;
630d49fc192SJustin Hibbits 
631d49fc192SJustin Hibbits 	rirq = priv;
632d49fc192SJustin Hibbits 
633d49fc192SJustin Hibbits 	if (!(rirq->flags & OPAL_XIVE_IRQ_LSI))
634d49fc192SJustin Hibbits 		return;
635d49fc192SJustin Hibbits 	xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_01);
636d49fc192SJustin Hibbits }
637d49fc192SJustin Hibbits 
638d49fc192SJustin Hibbits static void
639d49fc192SJustin Hibbits xive_unmask(device_t dev, u_int irq, void *priv)
640d49fc192SJustin Hibbits {
641d49fc192SJustin Hibbits 	struct xive_irq *rirq;
642d49fc192SJustin Hibbits 
643d49fc192SJustin Hibbits 	rirq = priv;
644d49fc192SJustin Hibbits 
645d49fc192SJustin Hibbits 	xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
646d49fc192SJustin Hibbits }
647d49fc192SJustin Hibbits 
648d49fc192SJustin Hibbits static void
649d49fc192SJustin Hibbits xive_translate_code(device_t dev, u_int irq, int code,
650d49fc192SJustin Hibbits     enum intr_trigger *trig, enum intr_polarity *pol)
651d49fc192SJustin Hibbits {
652d49fc192SJustin Hibbits 	switch (code) {
653d49fc192SJustin Hibbits 	case 0:
654d49fc192SJustin Hibbits 		/* L to H edge */
655d49fc192SJustin Hibbits 		*trig = INTR_TRIGGER_EDGE;
656d49fc192SJustin Hibbits 		*pol = INTR_POLARITY_HIGH;
657d49fc192SJustin Hibbits 		break;
658d49fc192SJustin Hibbits 	case 1:
659d49fc192SJustin Hibbits 		/* Active L level */
660d49fc192SJustin Hibbits 		*trig = INTR_TRIGGER_LEVEL;
661d49fc192SJustin Hibbits 		*pol = INTR_POLARITY_LOW;
662d49fc192SJustin Hibbits 		break;
663d49fc192SJustin Hibbits 	default:
664d49fc192SJustin Hibbits 		*trig = INTR_TRIGGER_CONFORM;
665d49fc192SJustin Hibbits 		*pol = INTR_POLARITY_CONFORM;
666d49fc192SJustin Hibbits 	}
667d49fc192SJustin Hibbits }
668d49fc192SJustin Hibbits 
669d49fc192SJustin Hibbits /* Private functions. */
670d49fc192SJustin Hibbits /*
671d49fc192SJustin Hibbits  * Setup the current CPU.  Called by the BSP at driver attachment, and by each
672d49fc192SJustin Hibbits  * AP at wakeup (via xive_smp_cpu_startup()).
673d49fc192SJustin Hibbits  */
674d49fc192SJustin Hibbits static void
675d49fc192SJustin Hibbits xive_setup_cpu(void)
676d49fc192SJustin Hibbits {
677d49fc192SJustin Hibbits 	struct xive_softc *sc;
678d49fc192SJustin Hibbits 	struct xive_cpu *cpup;
679d49fc192SJustin Hibbits 	uint32_t val;
680d49fc192SJustin Hibbits 
681d49fc192SJustin Hibbits 	cpup = DPCPU_PTR(xive_cpu_data);
682d49fc192SJustin Hibbits 
683d49fc192SJustin Hibbits 	sc = device_get_softc(root_pic);
684d49fc192SJustin Hibbits 
685d49fc192SJustin Hibbits 	val = bus_read_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2);
686d49fc192SJustin Hibbits 	if (val & TM_QW2W2_VP)
687d49fc192SJustin Hibbits 		bus_read_8(sc->sc_mem, XIVE_TM_SPC_PULL_POOL_CTX);
688d49fc192SJustin Hibbits 
689d49fc192SJustin Hibbits 	bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD0, 0xff);
690d49fc192SJustin Hibbits 	bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2,
691d49fc192SJustin Hibbits 	    TM_QW2W2_VP | cpup->cam);
692d49fc192SJustin Hibbits 
693d49fc192SJustin Hibbits 	xive_unmask(root_pic, cpup->ipi_data.girq, &cpup->ipi_data);
694d49fc192SJustin Hibbits 	xive_write_1(sc, XIVE_TM_CPPR, 0xff);
695d49fc192SJustin Hibbits }
696d49fc192SJustin Hibbits 
697d49fc192SJustin Hibbits /* Populate an IRQ structure, mapping the EOI and trigger pages. */
698d49fc192SJustin Hibbits static void
699d49fc192SJustin Hibbits xive_init_irq(struct xive_irq *irqd, u_int irq)
700d49fc192SJustin Hibbits {
701d49fc192SJustin Hibbits 	uint64_t eoi_phys, trig_phys;
702d49fc192SJustin Hibbits 	uint32_t esb_shift;
703d49fc192SJustin Hibbits 
704d49fc192SJustin Hibbits 	opal_call(OPAL_XIVE_GET_IRQ_INFO, irq,
705d49fc192SJustin Hibbits 	    vtophys(&irqd->flags), vtophys(&eoi_phys),
706d49fc192SJustin Hibbits 	    vtophys(&trig_phys), vtophys(&esb_shift),
707d49fc192SJustin Hibbits 	    vtophys(&irqd->chip));
708d49fc192SJustin Hibbits 
709f9acb7a8SBrandon Bergren 	irqd->flags = be64toh(irqd->flags);
710f9acb7a8SBrandon Bergren 	eoi_phys = be64toh(eoi_phys);
711f9acb7a8SBrandon Bergren 	trig_phys = be64toh(trig_phys);
712f9acb7a8SBrandon Bergren 	esb_shift = be32toh(esb_shift);
713f9acb7a8SBrandon Bergren 	irqd->chip = be32toh(irqd->chip);
714f9acb7a8SBrandon Bergren 
715d49fc192SJustin Hibbits 	irqd->girq = irq;
716d49fc192SJustin Hibbits 	irqd->esb_size = 1 << esb_shift;
717d49fc192SJustin Hibbits 	irqd->eoi_page = (vm_offset_t)pmap_mapdev(eoi_phys, irqd->esb_size);
718d49fc192SJustin Hibbits 
719d49fc192SJustin Hibbits 	if (eoi_phys == trig_phys)
720d49fc192SJustin Hibbits 		irqd->trig_page = irqd->eoi_page;
721d49fc192SJustin Hibbits 	else if (trig_phys != 0)
722d49fc192SJustin Hibbits 		irqd->trig_page = (vm_offset_t)pmap_mapdev(trig_phys,
723d49fc192SJustin Hibbits 		    irqd->esb_size);
724d49fc192SJustin Hibbits 	else
725d49fc192SJustin Hibbits 		irqd->trig_page = 0;
726d49fc192SJustin Hibbits 
727d49fc192SJustin Hibbits 	opal_call(OPAL_XIVE_GET_IRQ_CONFIG, irq, vtophys(&irqd->vp),
728d49fc192SJustin Hibbits 	    vtophys(&irqd->prio), vtophys(&irqd->lirq));
729f9acb7a8SBrandon Bergren 
730f9acb7a8SBrandon Bergren 	irqd->vp = be64toh(irqd->vp);
731f9acb7a8SBrandon Bergren 	irqd->prio = be64toh(irqd->prio);
732f9acb7a8SBrandon Bergren 	irqd->lirq = be32toh(irqd->lirq);
733d49fc192SJustin Hibbits }
734d49fc192SJustin Hibbits 
735d49fc192SJustin Hibbits /* Allocate an IRQ struct before populating it. */
736d49fc192SJustin Hibbits static struct xive_irq *
737d49fc192SJustin Hibbits xive_configure_irq(u_int irq)
738d49fc192SJustin Hibbits {
739d49fc192SJustin Hibbits 	struct xive_irq *irqd;
740d49fc192SJustin Hibbits 
741d49fc192SJustin Hibbits 	irqd = malloc(sizeof(struct xive_irq), M_XIVE, M_WAITOK);
742d49fc192SJustin Hibbits 
743d49fc192SJustin Hibbits 	xive_init_irq(irqd, irq);
744d49fc192SJustin Hibbits 
745d49fc192SJustin Hibbits 	return (irqd);
746d49fc192SJustin Hibbits }
747d49fc192SJustin Hibbits 
748d49fc192SJustin Hibbits /*
749d49fc192SJustin Hibbits  * Part of the OPAL API.  OPAL_XIVE_ALLOCATE_VP_BLOCK might require more pages,
750d49fc192SJustin Hibbits  * provisioned through this call.
751d49fc192SJustin Hibbits  */
752d49fc192SJustin Hibbits static int
753d49fc192SJustin Hibbits xive_provision_page(struct xive_softc *sc)
754d49fc192SJustin Hibbits {
755d49fc192SJustin Hibbits 	void *prov_page;
756d49fc192SJustin Hibbits 	int error;
757d49fc192SJustin Hibbits 
758d49fc192SJustin Hibbits 	do {
759d49fc192SJustin Hibbits 		prov_page = contigmalloc(sc->sc_prov_page_size, M_XIVE, 0,
760d49fc192SJustin Hibbits 		    0, BUS_SPACE_MAXADDR,
761d49fc192SJustin Hibbits 		    sc->sc_prov_page_size, sc->sc_prov_page_size);
762d49fc192SJustin Hibbits 
763d49fc192SJustin Hibbits 		error = opal_call(OPAL_XIVE_DONATE_PAGE, -1,
764d49fc192SJustin Hibbits 		    vtophys(prov_page));
765d49fc192SJustin Hibbits 	} while (error == OPAL_XIVE_PROVISIONING);
766d49fc192SJustin Hibbits 
767d49fc192SJustin Hibbits 	return (0);
768d49fc192SJustin Hibbits }
769d49fc192SJustin Hibbits 
770d49fc192SJustin Hibbits /* The XIVE_TM_CPPR register must be set by each thread */
771d49fc192SJustin Hibbits static void
772d49fc192SJustin Hibbits xive_smp_cpu_startup(void)
773d49fc192SJustin Hibbits {
774d49fc192SJustin Hibbits 
775d49fc192SJustin Hibbits 	xive_setup_cpu();
776d49fc192SJustin Hibbits }
777