xref: /freebsd/sys/powerpc/powernv/xive.c (revision f3c5273d315a64826d2149ac453ff8c4583ddbe8)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright 2019 Justin Hibbits
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_platform.h"
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/module.h>
36 #include <sys/bus.h>
37 #include <sys/conf.h>
38 #include <sys/endian.h>
39 #include <sys/kernel.h>
40 #include <sys/malloc.h>
41 #include <sys/smp.h>
42 
43 #include <vm/vm.h>
44 #include <vm/pmap.h>
45 
46 #include <machine/bus.h>
47 #include <machine/intr_machdep.h>
48 #include <machine/md_var.h>
49 
50 #include <dev/ofw/ofw_bus.h>
51 #include <dev/ofw/ofw_bus_subr.h>
52 
53 #ifdef POWERNV
54 #include <powerpc/powernv/opal.h>
55 #endif
56 
57 #include "pic_if.h"
58 
59 #define XIVE_PRIORITY	7	/* Random non-zero number */
60 #define MAX_XIVE_IRQS	(1<<24)	/* 24-bit XIRR field */
61 
62 /* Registers */
63 #define	XIVE_TM_QW1_OS		0x010	/* Guest OS registers */
64 #define	XIVE_TM_QW2_HV_POOL	0x020	/* Hypervisor pool registers */
65 #define	XIVE_TM_QW3_HV		0x030	/* Hypervisor registers */
66 
67 #define	XIVE_TM_NSR	0x00
68 #define	XIVE_TM_CPPR	0x01
69 #define	XIVE_TM_IPB	0x02
70 #define	XIVE_TM_LSMFB	0x03
71 #define	XIVE_TM_ACK_CNT	0x04
72 #define	XIVE_TM_INC	0x05
73 #define	XIVE_TM_AGE	0x06
74 #define	XIVE_TM_PIPR	0x07
75 
76 #define	TM_WORD0	0x0
77 #define	TM_WORD2	0x8
78 #define	  TM_QW2W2_VP	  0x80000000
79 
80 #define	XIVE_TM_SPC_ACK			0x800
81 #define	  TM_QW3NSR_HE_SHIFT		  14
82 #define	  TM_QW3_NSR_HE_NONE		  0
83 #define	  TM_QW3_NSR_HE_POOL		  1
84 #define	  TM_QW3_NSR_HE_PHYS		  2
85 #define	  TM_QW3_NSR_HE_LSI		  3
86 #define	XIVE_TM_SPC_PULL_POOL_CTX	0x828
87 
88 #define	XIVE_IRQ_LOAD_EOI	0x000
89 #define	XIVE_IRQ_STORE_EOI	0x400
90 #define	XIVE_IRQ_PQ_00		0xc00
91 #define	XIVE_IRQ_PQ_01		0xd00
92 
93 #define	XIVE_IRQ_VAL_P		0x02
94 #define	XIVE_IRQ_VAL_Q		0x01
95 
96 struct xive_softc;
97 struct xive_irq;
98 
99 extern void (*powernv_smp_ap_extra_init)(void);
100 
101 /* Private support */
102 static void	xive_setup_cpu(void);
103 static void	xive_smp_cpu_startup(void);
104 static void	xive_init_irq(struct xive_irq *irqd, u_int irq);
105 static struct xive_irq	*xive_configure_irq(u_int irq);
106 static int	xive_provision_page(struct xive_softc *sc);
107 
108 
109 /* Interfaces */
110 static int	xive_probe(device_t);
111 static int	xive_attach(device_t);
112 static int	xics_probe(device_t);
113 static int	xics_attach(device_t);
114 
115 static void	xive_bind(device_t, u_int, cpuset_t, void **);
116 static void	xive_dispatch(device_t, struct trapframe *);
117 static void	xive_enable(device_t, u_int, u_int, void **);
118 static void	xive_eoi(device_t, u_int, void *);
119 static void	xive_ipi(device_t, u_int);
120 static void	xive_mask(device_t, u_int, void *);
121 static void	xive_unmask(device_t, u_int, void *);
122 static void	xive_translate_code(device_t dev, u_int irq, int code,
123 		    enum intr_trigger *trig, enum intr_polarity *pol);
124 
125 static device_method_t  xive_methods[] = {
126 	/* Device interface */
127 	DEVMETHOD(device_probe,		xive_probe),
128 	DEVMETHOD(device_attach,	xive_attach),
129 
130 	/* PIC interface */
131 	DEVMETHOD(pic_bind,		xive_bind),
132 	DEVMETHOD(pic_dispatch,		xive_dispatch),
133 	DEVMETHOD(pic_enable,		xive_enable),
134 	DEVMETHOD(pic_eoi,		xive_eoi),
135 	DEVMETHOD(pic_ipi,		xive_ipi),
136 	DEVMETHOD(pic_mask,		xive_mask),
137 	DEVMETHOD(pic_unmask,		xive_unmask),
138 	DEVMETHOD(pic_translate_code,	xive_translate_code),
139 
140 	DEVMETHOD_END
141 };
142 
143 static device_method_t  xics_methods[] = {
144 	/* Device interface */
145 	DEVMETHOD(device_probe,		xics_probe),
146 	DEVMETHOD(device_attach,	xics_attach),
147 
148 	DEVMETHOD_END
149 };
150 
151 struct xive_softc {
152 	struct mtx sc_mtx;
153 	struct resource *sc_mem;
154 	vm_size_t	sc_prov_page_size;
155 	uint32_t	sc_offset;
156 };
157 
158 struct xive_queue {
159 	uint32_t	*q_page;
160 	uint32_t	*q_eoi_page;
161 	uint32_t	 q_toggle;
162 	uint32_t	 q_size;
163 	uint32_t	 q_index;
164 	uint32_t	 q_mask;
165 };
166 
167 struct xive_irq {
168 	uint32_t	girq;
169 	uint32_t	lirq;
170 	uint64_t	vp;
171 	uint64_t	flags;
172 #define	OPAL_XIVE_IRQ_EOI_VIA_FW	0x00000020
173 #define	OPAL_XIVE_IRQ_MASK_VIA_FW	0x00000010
174 #define	OPAL_XIVE_IRQ_SHIFT_BUG		0x00000008
175 #define	OPAL_XIVE_IRQ_LSI		0x00000004
176 #define	OPAL_XIVE_IRQ_STORE_EOI		0x00000002
177 #define	OPAL_XIVE_IRQ_TRIGGER_PAGE	0x00000001
178 	uint8_t	prio;
179 	vm_offset_t	eoi_page;
180 	vm_offset_t	trig_page;
181 	vm_size_t	esb_size;
182 	int		chip;
183 };
184 
185 struct xive_cpu {
186 	uint64_t	vp;
187 	uint64_t	flags;
188 	struct xive_irq	ipi_data;
189 	struct xive_queue	queue; /* We only use a single queue for now. */
190 	uint64_t	cam;
191 	uint32_t	chip;
192 };
193 
194 static driver_t xive_driver = {
195 	"xive",
196 	xive_methods,
197 	sizeof(struct xive_softc)
198 };
199 
200 static driver_t xics_driver = {
201 	"xivevc",
202 	xics_methods,
203 	0
204 };
205 
206 static devclass_t xive_devclass;
207 static devclass_t xics_devclass;
208 
209 EARLY_DRIVER_MODULE(xive, ofwbus, xive_driver, xive_devclass, 0, 0,
210     BUS_PASS_INTERRUPT-1);
211 EARLY_DRIVER_MODULE(xivevc, ofwbus, xics_driver, xics_devclass, 0, 0,
212     BUS_PASS_INTERRUPT);
213 
214 MALLOC_DEFINE(M_XIVE, "xive", "XIVE Memory");
215 
216 DPCPU_DEFINE_STATIC(struct xive_cpu, xive_cpu_data);
217 
218 static int xive_ipi_vector = -1;
219 
220 /*
221  * XIVE Exploitation mode driver.
222  *
223  * The XIVE, present in the POWER9 CPU, can run in two modes: XICS emulation
224  * mode, and "Exploitation mode".  XICS emulation mode is compatible with the
225  * POWER8 and earlier XICS interrupt controller, using OPAL calls to emulate
226  * hypervisor calls and memory accesses.  Exploitation mode gives us raw access
227  * to the XIVE MMIO, improving performance significantly.
228  *
229  * The XIVE controller is a very bizarre interrupt controller.  It uses queues
230  * in memory to pass interrupts around, and maps itself into 512GB of physical
231  * device address space, giving each interrupt in the system one or more pages
232  * of address space.  An IRQ is tied to a virtual processor, which could be a
233  * physical CPU thread, or a guest CPU thread (LPAR running on a physical
234  * thread).  Thus, the controller can route interrupts directly to guest OSes
235  * bypassing processing by the hypervisor, thereby improving performance of the
236  * guest OS.
237  *
238  * An IRQ, in addition to being tied to a virtual processor, has one or two
239  * page mappings: an EOI page, and an optional trigger page.  The trigger page
240  * could be the same as the EOI page.  Level-sensitive interrupts (LSIs) don't
241  * have a trigger page, as they're external interrupts controlled by physical
242  * lines.  MSIs and IPIs have trigger pages.  An IPI is really just another IRQ
243  * in the XIVE, which is triggered by software.
244  *
245  * An interesting behavior of the XIVE controller is that oftentimes the
246  * contents of an address location don't actually matter, but the direction of
247  * the action is the signifier (read vs write), and the address is significant.
248  * Hence, masking and unmasking an interrupt is done by reading different
249  * addresses in the EOI page, and triggering an interrupt consists of writing to
250  * the trigger page.
251  *
252  * Additionally, the MMIO region mapped is CPU-sensitive, just like the
253  * per-processor register space (private access) in OpenPIC.  In order for a CPU
254  * to receive interrupts it must itself configure its CPPR (Current Processor
255  * Priority Register), it cannot be set by any other processor.  This
256  * necessitates the xive_smp_cpu_startup() function.
257  *
258  * Queues are pages of memory, sized powers-of-two, that are shared with the
259  * XIVE.  The XIVE writes into the queue with an alternating polarity bit, which
260  * flips when the queue wraps.
261  */
262 
263 /*
264  * Offset-based read/write interfaces.
265  */
266 static uint16_t
267 xive_read_2(struct xive_softc *sc, bus_size_t offset)
268 {
269 
270 	return (bus_read_2(sc->sc_mem, sc->sc_offset + offset));
271 }
272 
273 static void
274 xive_write_1(struct xive_softc *sc, bus_size_t offset, uint8_t val)
275 {
276 
277 	bus_write_1(sc->sc_mem, sc->sc_offset + offset, val);
278 }
279 
280 /* EOI and Trigger page access interfaces. */
281 static uint64_t
282 xive_read_mmap8(vm_offset_t addr)
283 {
284 	return (*(volatile uint64_t *)addr);
285 }
286 
287 static void
288 xive_write_mmap8(vm_offset_t addr, uint64_t val)
289 {
290 	*(uint64_t *)(addr) = val;
291 }
292 
293 
294 /* Device interfaces. */
295 static int
296 xive_probe(device_t dev)
297 {
298 
299 	if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-pe"))
300 		return (ENXIO);
301 
302 	device_set_desc(dev, "External Interrupt Virtualization Engine");
303 
304 	/* Make sure we always win against the xicp driver. */
305 	return (BUS_PROBE_DEFAULT);
306 }
307 
308 static int
309 xics_probe(device_t dev)
310 {
311 
312 	if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-vc"))
313 		return (ENXIO);
314 
315 	device_set_desc(dev, "External Interrupt Virtualization Engine Root");
316 	return (BUS_PROBE_DEFAULT);
317 }
318 
319 static int
320 xive_attach(device_t dev)
321 {
322 	struct xive_softc *sc = device_get_softc(dev);
323 	struct xive_cpu *xive_cpud;
324 	phandle_t phandle = ofw_bus_get_node(dev);
325 	int64_t vp_block;
326 	int error;
327 	int rid;
328 	int i, order;
329 	uint64_t vp_id;
330 	int64_t ipi_irq;
331 
332 	opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EXP);
333 
334 	error = OF_getencprop(phandle, "ibm,xive-provision-page-size",
335 	    (pcell_t *)&sc->sc_prov_page_size, sizeof(sc->sc_prov_page_size));
336 
337 	rid = 1;	/* Get the Hypervisor-level register set. */
338 	sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
339 	    &rid, RF_ACTIVE);
340 	sc->sc_offset = XIVE_TM_QW3_HV;
341 
342 	mtx_init(&sc->sc_mtx, "XIVE", NULL, MTX_DEF);
343 
344 	order = fls(mp_maxid + (mp_maxid - 1)) - 1;
345 
346 	do {
347 		vp_block = opal_call(OPAL_XIVE_ALLOCATE_VP_BLOCK, order);
348 		if (vp_block == OPAL_BUSY)
349 			DELAY(10);
350 		else if (vp_block == OPAL_XIVE_PROVISIONING)
351 			xive_provision_page(sc);
352 		else
353 			break;
354 	} while (1);
355 
356 	if (vp_block < 0) {
357 		device_printf(dev,
358 		    "Unable to allocate VP block.  Opal error %d\n",
359 		    (int)vp_block);
360 		bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
361 		return (ENXIO);
362 	}
363 
364 	/*
365 	 * Set up the VPs.  Try to do as much as we can in attach, to lessen
366 	 * what's needed at AP spawn time.
367 	 */
368 	CPU_FOREACH(i) {
369 		vp_id = pcpu_find(i)->pc_hwref;
370 
371 		xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data);
372 		xive_cpud->vp = vp_id + vp_block;
373 		opal_call(OPAL_XIVE_GET_VP_INFO, xive_cpud->vp, NULL,
374 		    vtophys(&xive_cpud->cam), NULL, vtophys(&xive_cpud->chip));
375 
376 		/* Allocate the queue page and populate the queue state data. */
377 		xive_cpud->queue.q_page = contigmalloc(PAGE_SIZE, M_XIVE,
378 		    M_ZERO | M_WAITOK, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
379 		xive_cpud->queue.q_size = 1 << PAGE_SHIFT;
380 		xive_cpud->queue.q_mask =
381 		    ((xive_cpud->queue.q_size / sizeof(int)) - 1);
382 		xive_cpud->queue.q_toggle = 0;
383 		xive_cpud->queue.q_index = 0;
384 		do {
385 			error = opal_call(OPAL_XIVE_SET_VP_INFO, xive_cpud->vp,
386 			    OPAL_XIVE_VP_ENABLED, 0);
387 		} while (error == OPAL_BUSY);
388 		error = opal_call(OPAL_XIVE_SET_QUEUE_INFO, vp_id,
389 		    XIVE_PRIORITY, vtophys(xive_cpud->queue.q_page), PAGE_SHIFT,
390 		    OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED);
391 
392 		do {
393 			ipi_irq = opal_call(OPAL_XIVE_ALLOCATE_IRQ,
394 			    xive_cpud->chip);
395 		} while (ipi_irq == OPAL_BUSY);
396 
397 		if (ipi_irq < 0)
398 			device_printf(root_pic,
399 			    "Failed allocating IPI.  OPAL error %d\n",
400 			    (int)ipi_irq);
401 		else {
402 			xive_init_irq(&xive_cpud->ipi_data, ipi_irq);
403 			xive_cpud->ipi_data.vp = vp_id;
404 			xive_cpud->ipi_data.lirq = MAX_XIVE_IRQS;
405 			opal_call(OPAL_XIVE_SET_IRQ_CONFIG, ipi_irq,
406 			    xive_cpud->ipi_data.vp, XIVE_PRIORITY,
407 			    MAX_XIVE_IRQS);
408 		}
409 	}
410 
411 	powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XIVE_IRQS,
412 	    1 /* Number of IPIs */, FALSE);
413 	root_pic = dev;
414 
415 	xive_setup_cpu();
416 	powernv_smp_ap_extra_init = xive_smp_cpu_startup;
417 
418 	return (0);
419 }
420 
421 static int
422 xics_attach(device_t dev)
423 {
424 	phandle_t phandle = ofw_bus_get_node(dev);
425 
426 	/* The XIVE (root PIC) will handle all our interrupts */
427 	powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
428 	    MAX_XIVE_IRQS, 1 /* Number of IPIs */, FALSE);
429 
430 	return (0);
431 }
432 
433 /*
434  * PIC I/F methods.
435  */
436 
437 static void
438 xive_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
439 {
440 	struct xive_irq *irqd;
441 	int cpu;
442 	int ncpus, i, error;
443 
444 	if (*priv == NULL)
445 		*priv = xive_configure_irq(irq);
446 
447 	irqd = *priv;
448 
449 	/*
450 	 * This doesn't appear to actually support affinity groups, so pick a
451 	 * random CPU.
452 	 */
453 	ncpus = 0;
454 	CPU_FOREACH(cpu)
455 		if (CPU_ISSET(cpu, &cpumask)) ncpus++;
456 
457 	i = mftb() % ncpus;
458 	ncpus = 0;
459 	CPU_FOREACH(cpu) {
460 		if (!CPU_ISSET(cpu, &cpumask))
461 			continue;
462 		if (ncpus == i)
463 			break;
464 		ncpus++;
465 	}
466 
467 	opal_call(OPAL_XIVE_SYNC);
468 
469 	irqd->vp = pcpu_find(cpu)->pc_hwref;
470 	error = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, irqd->vp,
471 	    XIVE_PRIORITY, irqd->lirq);
472 
473 	if (error < 0)
474 		panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
475 
476 	xive_eoi(dev, irq, irqd);
477 }
478 
479 /* Read the next entry in the queue page and update the index. */
480 static int
481 xive_read_eq(struct xive_queue *q)
482 {
483 	uint32_t i = be32toh(q->q_page[q->q_index]);
484 
485 	/* Check validity, using current queue polarity. */
486 	if ((i >> 31) == q->q_toggle)
487 		return (0);
488 
489 	q->q_index = (q->q_index + 1) & q->q_mask;
490 
491 	if (q->q_index == 0)
492 		q->q_toggle ^= 1;
493 
494 	return (i & 0x7fffffff);
495 }
496 
497 static void
498 xive_dispatch(device_t dev, struct trapframe *tf)
499 {
500 	struct xive_softc *sc;
501 	struct xive_cpu *xive_cpud;
502 	uint32_t vector;
503 	uint16_t ack;
504 	uint8_t cppr, he;
505 
506 	sc = device_get_softc(dev);
507 
508 	for (;;) {
509 		ack = xive_read_2(sc, XIVE_TM_SPC_ACK);
510 		cppr = (ack & 0xff);
511 
512 		he = ack >> TM_QW3NSR_HE_SHIFT;
513 
514 		if (he == TM_QW3_NSR_HE_NONE)
515 			break;
516 		switch (he) {
517 		case TM_QW3_NSR_HE_NONE:
518 			goto end;
519 		case TM_QW3_NSR_HE_POOL:
520 		case TM_QW3_NSR_HE_LSI:
521 			device_printf(dev,
522 			    "Unexpected interrupt he type: %d\n", he);
523 			goto end;
524 		case TM_QW3_NSR_HE_PHYS:
525 			break;
526 		}
527 
528 		xive_cpud = DPCPU_PTR(xive_cpu_data);
529 		xive_write_1(sc, XIVE_TM_CPPR, cppr);
530 
531 		for (;;) {
532 			vector = xive_read_eq(&xive_cpud->queue);
533 
534 			if (vector == 0)
535 				break;
536 
537 			if (vector == MAX_XIVE_IRQS)
538 				vector = xive_ipi_vector;
539 
540 			powerpc_dispatch_intr(vector, tf);
541 		}
542 	}
543 end:
544 	xive_write_1(sc, XIVE_TM_CPPR, 0xff);
545 }
546 
547 static void
548 xive_enable(device_t dev, u_int irq, u_int vector, void **priv)
549 {
550 	struct xive_irq *irqd;
551 	cell_t status, cpu;
552 
553 	if (irq == MAX_XIVE_IRQS) {
554 		if (xive_ipi_vector == -1)
555 			xive_ipi_vector = vector;
556 		return;
557 	}
558 	if (*priv == NULL)
559 		*priv = xive_configure_irq(irq);
560 
561 	irqd = *priv;
562 
563 	/* Bind to this CPU to start */
564 	cpu = PCPU_GET(hwref);
565 	irqd->lirq = vector;
566 
567 	for (;;) {
568 		status = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, cpu,
569 		    XIVE_PRIORITY, vector);
570 		if (status != OPAL_BUSY)
571 			break;
572 		DELAY(10);
573 	}
574 
575 	if (status != 0)
576 		panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
577 		    cpu, status);
578 
579 	xive_unmask(dev, irq, *priv);
580 }
581 
582 static void
583 xive_eoi(device_t dev, u_int irq, void *priv)
584 {
585 	struct xive_irq *rirq;
586 	struct xive_cpu *cpud;
587 	uint8_t eoi_val;
588 
589 	if (irq == MAX_XIVE_IRQS) {
590 		cpud = DPCPU_PTR(xive_cpu_data);
591 		rirq = &cpud->ipi_data;
592 	} else
593 		rirq = priv;
594 
595 	if (rirq->flags & OPAL_XIVE_IRQ_EOI_VIA_FW)
596 		opal_call(OPAL_INT_EOI, irq);
597 	else if (rirq->flags & OPAL_XIVE_IRQ_STORE_EOI)
598 		xive_write_mmap8(rirq->eoi_page + XIVE_IRQ_STORE_EOI, 0);
599 	else if (rirq->flags & OPAL_XIVE_IRQ_LSI)
600 		xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_LOAD_EOI);
601 	else {
602 		eoi_val = xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
603 		if ((eoi_val & XIVE_IRQ_VAL_Q) && rirq->trig_page != 0)
604 			xive_write_mmap8(rirq->trig_page, 0);
605 	}
606 }
607 
608 static void
609 xive_ipi(device_t dev, u_int cpu)
610 {
611 	struct xive_cpu *xive_cpud;
612 
613 	xive_cpud = DPCPU_ID_PTR(cpu, xive_cpu_data);
614 
615 	if (xive_cpud->ipi_data.trig_page == 0)
616 		return;
617 	xive_write_mmap8(xive_cpud->ipi_data.trig_page, 0);
618 }
619 
620 static void
621 xive_mask(device_t dev, u_int irq, void *priv)
622 {
623 	struct xive_irq *rirq;
624 
625 	/* Never mask IPIs */
626 	if (irq == MAX_XIVE_IRQS)
627 		return;
628 
629 	rirq = priv;
630 
631 	if (!(rirq->flags & OPAL_XIVE_IRQ_LSI))
632 		return;
633 	xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_01);
634 }
635 
636 static void
637 xive_unmask(device_t dev, u_int irq, void *priv)
638 {
639 	struct xive_irq *rirq;
640 
641 	rirq = priv;
642 
643 	xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
644 }
645 
646 static void
647 xive_translate_code(device_t dev, u_int irq, int code,
648     enum intr_trigger *trig, enum intr_polarity *pol)
649 {
650 	switch (code) {
651 	case 0:
652 		/* L to H edge */
653 		*trig = INTR_TRIGGER_EDGE;
654 		*pol = INTR_POLARITY_HIGH;
655 		break;
656 	case 1:
657 		/* Active L level */
658 		*trig = INTR_TRIGGER_LEVEL;
659 		*pol = INTR_POLARITY_LOW;
660 		break;
661 	default:
662 		*trig = INTR_TRIGGER_CONFORM;
663 		*pol = INTR_POLARITY_CONFORM;
664 	}
665 }
666 
667 /* Private functions. */
668 /*
669  * Setup the current CPU.  Called by the BSP at driver attachment, and by each
670  * AP at wakeup (via xive_smp_cpu_startup()).
671  */
672 static void
673 xive_setup_cpu(void)
674 {
675 	struct xive_softc *sc;
676 	struct xive_cpu *cpup;
677 	uint32_t val;
678 
679 	cpup = DPCPU_PTR(xive_cpu_data);
680 
681 	sc = device_get_softc(root_pic);
682 
683 	val = bus_read_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2);
684 	if (val & TM_QW2W2_VP)
685 		bus_read_8(sc->sc_mem, XIVE_TM_SPC_PULL_POOL_CTX);
686 
687 	bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD0, 0xff);
688 	bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2,
689 	    TM_QW2W2_VP | cpup->cam);
690 
691 	xive_unmask(root_pic, cpup->ipi_data.girq, &cpup->ipi_data);
692 	xive_write_1(sc, XIVE_TM_CPPR, 0xff);
693 }
694 
695 /* Populate an IRQ structure, mapping the EOI and trigger pages. */
696 static void
697 xive_init_irq(struct xive_irq *irqd, u_int irq)
698 {
699 	uint64_t eoi_phys, trig_phys;
700 	uint32_t esb_shift;
701 
702 	opal_call(OPAL_XIVE_GET_IRQ_INFO, irq,
703 	    vtophys(&irqd->flags), vtophys(&eoi_phys),
704 	    vtophys(&trig_phys), vtophys(&esb_shift),
705 	    vtophys(&irqd->chip));
706 
707 	irqd->girq = irq;
708 	irqd->esb_size = 1 << esb_shift;
709 	irqd->eoi_page = (vm_offset_t)pmap_mapdev(eoi_phys, irqd->esb_size);
710 
711 	if (eoi_phys == trig_phys)
712 		irqd->trig_page = irqd->eoi_page;
713 	else if (trig_phys != 0)
714 		irqd->trig_page = (vm_offset_t)pmap_mapdev(trig_phys,
715 		    irqd->esb_size);
716 	else
717 		irqd->trig_page = 0;
718 
719 	opal_call(OPAL_XIVE_GET_IRQ_CONFIG, irq, vtophys(&irqd->vp),
720 	    vtophys(&irqd->prio), vtophys(&irqd->lirq));
721 }
722 
723 /* Allocate an IRQ struct before populating it. */
724 static struct xive_irq *
725 xive_configure_irq(u_int irq)
726 {
727 	struct xive_irq *irqd;
728 
729 	irqd = malloc(sizeof(struct xive_irq), M_XIVE, M_WAITOK);
730 
731 	xive_init_irq(irqd, irq);
732 
733 	return (irqd);
734 }
735 
736 /*
737  * Part of the OPAL API.  OPAL_XIVE_ALLOCATE_VP_BLOCK might require more pages,
738  * provisioned through this call.
739  */
740 static int
741 xive_provision_page(struct xive_softc *sc)
742 {
743 	void *prov_page;
744 	int error;
745 
746 	do {
747 		prov_page = contigmalloc(sc->sc_prov_page_size, M_XIVE, 0,
748 		    0, BUS_SPACE_MAXADDR,
749 		    sc->sc_prov_page_size, sc->sc_prov_page_size);
750 
751 		error = opal_call(OPAL_XIVE_DONATE_PAGE, -1,
752 		    vtophys(prov_page));
753 	} while (error == OPAL_XIVE_PROVISIONING);
754 
755 	return (0);
756 }
757 
758 /* The XIVE_TM_CPPR register must be set by each thread */
759 static void
760 xive_smp_cpu_startup(void)
761 {
762 
763 	xive_setup_cpu();
764 }
765