xref: /freebsd/sys/powerpc/pseries/xics.c (revision db70ff37a051dfa19f6f3f0f0c5e3571aba91982)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright 2011 Nathan Whitehorn
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_platform.h"
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/module.h>
36 #include <sys/bus.h>
37 #include <sys/conf.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/smp.h>
41 
42 #include <vm/vm.h>
43 #include <vm/pmap.h>
44 
45 #include <machine/bus.h>
46 #include <machine/intr_machdep.h>
47 #include <machine/md_var.h>
48 #include <machine/rtas.h>
49 
50 #include <dev/ofw/ofw_bus.h>
51 #include <dev/ofw/ofw_bus_subr.h>
52 
53 #ifdef POWERNV
54 #include <powerpc/powernv/opal.h>
55 #endif
56 
57 #include "phyp-hvcall.h"
58 #include "pic_if.h"
59 
60 #define XICP_PRIORITY	5	/* Random non-zero number */
61 #define XICP_IPI	2
62 #define MAX_XICP_IRQS	(1<<24)	/* 24-bit XIRR field */
63 
64 #define	XIVE_XICS_MODE_EMU	0
65 #define	XIVE_XICS_MODE_EXP	1
66 
67 static int	xicp_probe(device_t);
68 static int	xicp_attach(device_t);
69 static int	xics_probe(device_t);
70 static int	xics_attach(device_t);
71 
72 static void	xicp_bind(device_t dev, u_int irq, cpuset_t cpumask);
73 static void	xicp_dispatch(device_t, struct trapframe *);
74 static void	xicp_enable(device_t, u_int, u_int);
75 static void	xicp_eoi(device_t, u_int);
76 static void	xicp_ipi(device_t, u_int);
77 static void	xicp_mask(device_t, u_int);
78 static void	xicp_unmask(device_t, u_int);
79 
80 #ifdef POWERNV
81 void	xicp_smp_cpu_startup(void);
82 #endif
83 
84 static device_method_t  xicp_methods[] = {
85 	/* Device interface */
86 	DEVMETHOD(device_probe,		xicp_probe),
87 	DEVMETHOD(device_attach,	xicp_attach),
88 
89 	/* PIC interface */
90 	DEVMETHOD(pic_bind,		xicp_bind),
91 	DEVMETHOD(pic_dispatch,		xicp_dispatch),
92 	DEVMETHOD(pic_enable,		xicp_enable),
93 	DEVMETHOD(pic_eoi,		xicp_eoi),
94 	DEVMETHOD(pic_ipi,		xicp_ipi),
95 	DEVMETHOD(pic_mask,		xicp_mask),
96 	DEVMETHOD(pic_unmask,		xicp_unmask),
97 
98 	DEVMETHOD_END
99 };
100 
101 static device_method_t  xics_methods[] = {
102 	/* Device interface */
103 	DEVMETHOD(device_probe,		xics_probe),
104 	DEVMETHOD(device_attach,	xics_attach),
105 
106 	DEVMETHOD_END
107 };
108 
109 struct xicp_softc {
110 	struct mtx sc_mtx;
111 	struct resource *mem[MAXCPU];
112 
113 	int cpu_range[2];
114 
115 	int ibm_int_on;
116 	int ibm_int_off;
117 	int ibm_get_xive;
118 	int ibm_set_xive;
119 
120 	/* XXX: inefficient -- hash table? tree? */
121 	struct {
122 		int irq;
123 		int vector;
124 		int cpu;
125 	} intvecs[256];
126 	int nintvecs;
127 	bool xics_emu;
128 };
129 
130 static driver_t xicp_driver = {
131 	"xicp",
132 	xicp_methods,
133 	sizeof(struct xicp_softc)
134 };
135 
136 static driver_t xics_driver = {
137 	"xics",
138 	xics_methods,
139 	0
140 };
141 
142 #ifdef POWERNV
143 /* We can only pass physical addresses into OPAL.  Kernel stacks are in the KVA,
144  * not in the direct map, so we need to somehow extract the physical address.
145  * However, pmap_kextract() takes locks, which is forbidden in a critical region
146  * (which PIC_DISPATCH() operates in).  The kernel is mapped into the Direct
147  * Map (0xc000....), and the CPU implicitly drops the top two bits when doing
148  * real address by nature that the bus width is smaller than 64-bits.  Placing
149  * cpu_xirr into the DMAP lets us take advantage of this and avoids the
150  * pmap_kextract() that would otherwise be needed if using the stack variable.
151  */
152 static uint32_t cpu_xirr[MAXCPU];
153 #endif
154 
155 static devclass_t xicp_devclass;
156 static devclass_t xics_devclass;
157 
158 EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, xicp_devclass, 0, 0,
159     BUS_PASS_INTERRUPT-1);
160 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, xics_devclass, 0, 0,
161     BUS_PASS_INTERRUPT);
162 
163 #ifdef POWERNV
164 static struct resource *
165 xicp_mem_for_cpu(int cpu)
166 {
167 	device_t dev;
168 	struct xicp_softc *sc;
169 	int i;
170 
171 	for (i = 0; (dev = devclass_get_device(xicp_devclass, i)) != NULL; i++){
172 		sc = device_get_softc(dev);
173 		if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1])
174 			return (sc->mem[cpu - sc->cpu_range[0]]);
175 	}
176 
177 	return (NULL);
178 }
179 #endif
180 
181 static int
182 xicp_probe(device_t dev)
183 {
184 
185 	if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") &&
186 	    !ofw_bus_is_compatible(dev, "ibm,opal-intc"))
187 		return (ENXIO);
188 
189 	device_set_desc(dev, "External Interrupt Presentation Controller");
190 	return (BUS_PROBE_GENERIC);
191 }
192 
193 static int
194 xics_probe(device_t dev)
195 {
196 
197 	if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") &&
198 	    !ofw_bus_is_compatible(dev, "IBM,opal-xics"))
199 		return (ENXIO);
200 
201 	device_set_desc(dev, "External Interrupt Source Controller");
202 	return (BUS_PROBE_GENERIC);
203 }
204 
205 static int
206 xicp_attach(device_t dev)
207 {
208 	struct xicp_softc *sc = device_get_softc(dev);
209 	phandle_t phandle = ofw_bus_get_node(dev);
210 
211 	if (rtas_exists()) {
212 		sc->ibm_int_on = rtas_token_lookup("ibm,int-on");
213 		sc->ibm_int_off = rtas_token_lookup("ibm,int-off");
214 		sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive");
215 		sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive");
216 #ifdef POWERNV
217 	} else if (opal_check() == 0) {
218 		/* No init needed */
219 #endif
220 	} else {
221 		device_printf(dev, "Cannot attach without RTAS or OPAL\n");
222 		return (ENXIO);
223 	}
224 
225 	if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) {
226 		OF_getencprop(phandle, "ibm,interrupt-server-ranges",
227 		    sc->cpu_range, sizeof(sc->cpu_range));
228 		sc->cpu_range[1] += sc->cpu_range[0];
229 		device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0],
230 		    sc->cpu_range[1]-1);
231 #ifdef POWERNV
232 	} else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) {
233 			/*
234 			 * For now run POWER9 XIVE interrupt controller in XICS
235 			 * compatibility mode.
236 			 */
237 			sc->xics_emu = true;
238 			opal_call(OPAL_XIVE_RESET, XIVE_XICS_MODE_EMU);
239 #endif
240 	} else {
241 		sc->cpu_range[0] = 0;
242 		sc->cpu_range[1] = mp_ncpus;
243 	}
244 
245 #ifdef POWERNV
246 	if (mfmsr() & PSL_HV) {
247 		int i;
248 
249 		if (sc->xics_emu) {
250 			opal_call(OPAL_INT_SET_CPPR, 0xff);
251 			for (i = 0; i < mp_ncpus; i++) {
252 				opal_call(OPAL_INT_SET_MFRR,
253 				    pcpu_find(i)->pc_hwref, 0xff);
254 			}
255 		} else {
256 			for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) {
257 				sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
258 				    &i, RF_ACTIVE);
259 				if (sc->mem[i] == NULL) {
260 					device_printf(dev, "Could not alloc mem "
261 					    "resource %d\n", i);
262 					return (ENXIO);
263 				}
264 
265 				/* Unmask interrupts on all cores */
266 				bus_write_1(sc->mem[i], 4, 0xff);
267 				bus_write_1(sc->mem[i], 12, 0xff);
268 			}
269 		}
270 	}
271 #endif
272 
273 	mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF);
274 	sc->nintvecs = 0;
275 
276 	powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS,
277 	    1 /* Number of IPIs */, FALSE);
278 	root_pic = dev;
279 
280 	return (0);
281 }
282 
283 static int
284 xics_attach(device_t dev)
285 {
286 	phandle_t phandle = ofw_bus_get_node(dev);
287 
288 	/* The XICP (root PIC) will handle all our interrupts */
289 	powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
290 	    MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE);
291 
292 	return (0);
293 }
294 
295 /*
296  * PIC I/F methods.
297  */
298 
299 static void
300 xicp_bind(device_t dev, u_int irq, cpuset_t cpumask)
301 {
302 	struct xicp_softc *sc = device_get_softc(dev);
303 	cell_t status, cpu;
304 	int ncpus, i, error;
305 
306 	/* Ignore IPIs */
307 	if (irq == MAX_XICP_IRQS)
308 		return;
309 
310 	/*
311 	 * This doesn't appear to actually support affinity groups, so pick a
312 	 * random CPU.
313 	 */
314 	ncpus = 0;
315 	CPU_FOREACH(cpu)
316 		if (CPU_ISSET(cpu, &cpumask)) ncpus++;
317 
318 	i = mftb() % ncpus;
319 	ncpus = 0;
320 	CPU_FOREACH(cpu) {
321 		if (!CPU_ISSET(cpu, &cpumask))
322 			continue;
323 		if (ncpus == i)
324 			break;
325 		ncpus++;
326 	}
327 
328 	cpu = pcpu_find(cpu)->pc_hwref;
329 
330 	/* XXX: super inefficient */
331 	for (i = 0; i < sc->nintvecs; i++) {
332 		if (sc->intvecs[i].irq == irq) {
333 			sc->intvecs[i].cpu = cpu;
334 			break;
335 		}
336 	}
337 	KASSERT(i < sc->nintvecs, ("Binding non-configured interrupt"));
338 
339 	if (rtas_exists())
340 		error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
341 		    XICP_PRIORITY, &status);
342 #ifdef POWERNV
343 	else
344 		error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
345 #endif
346 
347 	if (error < 0)
348 		panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
349 }
350 
351 static void
352 xicp_dispatch(device_t dev, struct trapframe *tf)
353 {
354 	struct xicp_softc *sc;
355 	struct resource *regs = NULL;
356 	uint64_t xirr, junk;
357 	int i;
358 
359 	sc = device_get_softc(dev);
360 #ifdef POWERNV
361 	if ((mfmsr() & PSL_HV) && !sc->xics_emu) {
362 		regs = xicp_mem_for_cpu(PCPU_GET(hwref));
363 		KASSERT(regs != NULL,
364 		    ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref)));
365 	}
366 #endif
367 
368 	for (;;) {
369 		/* Return value in R4, use the PFT call */
370 		if (regs) {
371 			xirr = bus_read_4(regs, 4);
372 #ifdef POWERNV
373 		} else if (sc->xics_emu) {
374 			opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)],
375 			    false);
376 			xirr = cpu_xirr[PCPU_GET(cpuid)];
377 #endif
378 		} else {
379 			/* Return value in R4, use the PFT call */
380 			phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk);
381 		}
382 		xirr &= 0x00ffffff;
383 
384 		if (xirr == 0) /* No more pending interrupts? */
385 			break;
386 
387 		if (xirr == XICP_IPI) {		/* Magic number for IPIs */
388 			xirr = MAX_XICP_IRQS;	/* Map to FreeBSD magic */
389 
390 			/* Clear IPI */
391 			if (regs)
392 				bus_write_1(regs, 12, 0xff);
393 #ifdef POWERNV
394 			else if (sc->xics_emu)
395 				opal_call(OPAL_INT_SET_MFRR,
396 				    PCPU_GET(hwref), 0xff);
397 #endif
398 			else
399 				phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)),
400 				    0xff);
401 		}
402 
403 		/* XXX: super inefficient */
404 		for (i = 0; i < sc->nintvecs; i++) {
405 			if (sc->intvecs[i].irq == xirr)
406 				break;
407 		}
408 
409 		KASSERT(i < sc->nintvecs, ("Unmapped XIRR"));
410 		powerpc_dispatch_intr(sc->intvecs[i].vector, tf);
411 	}
412 }
413 
414 static void
415 xicp_enable(device_t dev, u_int irq, u_int vector)
416 {
417 	struct xicp_softc *sc;
418 	cell_t status, cpu;
419 
420 	sc = device_get_softc(dev);
421 
422 	KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs),
423 		("Too many XICP interrupts"));
424 
425 	/* Bind to this CPU to start: distrib. ID is last entry in gserver# */
426 	cpu = PCPU_GET(hwref);
427 
428 	mtx_lock(&sc->sc_mtx);
429 	sc->intvecs[sc->nintvecs].irq = irq;
430 	sc->intvecs[sc->nintvecs].vector = vector;
431 	sc->intvecs[sc->nintvecs].cpu = cpu;
432 	mb();
433 	sc->nintvecs++;
434 	mtx_unlock(&sc->sc_mtx);
435 
436 	/* IPIs are also enabled */
437 	if (irq == MAX_XICP_IRQS)
438 		return;
439 
440 	if (rtas_exists()) {
441 		rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
442 		    XICP_PRIORITY, &status);
443 		xicp_unmask(dev, irq);
444 #ifdef POWERNV
445 	} else {
446 		status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
447 		/* Unmask implicit for OPAL */
448 
449 		if (status != 0)
450 			panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
451 			    cpu, status);
452 #endif
453 	}
454 }
455 
456 static void
457 xicp_eoi(device_t dev, u_int irq)
458 {
459 #ifdef POWERNV
460 	struct xicp_softc *sc;
461 #endif
462 	uint64_t xirr;
463 
464 	if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */
465 		irq = XICP_IPI;
466 	xirr = irq | (0xff << 24);
467 
468 #ifdef POWERNV
469 	if (mfmsr() & PSL_HV) {
470 		sc = device_get_softc(dev);
471 		if (sc->xics_emu)
472 			opal_call(OPAL_INT_EOI, xirr);
473 		else
474 			bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr);
475 	} else
476 #endif
477 		phyp_hcall(H_EOI, xirr);
478 }
479 
480 static void
481 xicp_ipi(device_t dev, u_int cpu)
482 {
483 
484 #ifdef POWERNV
485 	struct xicp_softc *sc;
486 	cpu = pcpu_find(cpu)->pc_hwref;
487 
488 	if (mfmsr() & PSL_HV) {
489 		sc = device_get_softc(dev);
490 		if (sc->xics_emu) {
491 			int64_t rv;
492 			rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY);
493 			if (rv != 0)
494 			    device_printf(dev, "IPI SET_MFRR result: %ld\n", rv);
495 		} else
496 			bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY);
497 	} else
498 #endif
499 		phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY);
500 }
501 
502 static void
503 xicp_mask(device_t dev, u_int irq)
504 {
505 	struct xicp_softc *sc = device_get_softc(dev);
506 	cell_t status;
507 
508 	if (irq == MAX_XICP_IRQS)
509 		return;
510 
511 	if (rtas_exists()) {
512 		rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status);
513 #ifdef POWERNV
514 	} else {
515 		int i;
516 
517 		for (i = 0; i < sc->nintvecs; i++) {
518 			if (sc->intvecs[i].irq == irq) {
519 				break;
520 			}
521 		}
522 		KASSERT(i < sc->nintvecs, ("Masking unconfigured interrupt"));
523 		opal_call(OPAL_SET_XIVE, irq, sc->intvecs[i].cpu << 2, 0xff);
524 #endif
525 	}
526 }
527 
528 static void
529 xicp_unmask(device_t dev, u_int irq)
530 {
531 	struct xicp_softc *sc = device_get_softc(dev);
532 	cell_t status;
533 
534 	if (irq == MAX_XICP_IRQS)
535 		return;
536 
537 	if (rtas_exists()) {
538 		rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status);
539 #ifdef POWERNV
540 	} else {
541 		int i;
542 
543 		for (i = 0; i < sc->nintvecs; i++) {
544 			if (sc->intvecs[i].irq == irq) {
545 				break;
546 			}
547 		}
548 		KASSERT(i < sc->nintvecs, ("Unmasking unconfigured interrupt"));
549 		opal_call(OPAL_SET_XIVE, irq, sc->intvecs[i].cpu << 2,
550 		    XICP_PRIORITY);
551 #endif
552 	}
553 }
554 
555 #ifdef POWERNV
556 /* This is only used on POWER9 systems with the XIVE's XICS emulation. */
557 void
558 xicp_smp_cpu_startup(void)
559 {
560 	struct xicp_softc *sc;
561 
562 	if (mfmsr() & PSL_HV) {
563 		sc = device_get_softc(root_pic);
564 
565 		if (sc->xics_emu)
566 			opal_call(OPAL_INT_SET_CPPR, 0xff);
567 	}
568 }
569 #endif
570