xref: /freebsd/sys/powerpc/pseries/xics.c (revision 2f513db72b034fd5ef7f080b11be5c711c15186a)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright 2011 Nathan Whitehorn
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_platform.h"
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/module.h>
36 #include <sys/bus.h>
37 #include <sys/conf.h>
38 #include <sys/kernel.h>
39 #include <sys/lock.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/smp.h>
43 
44 #include <vm/vm.h>
45 #include <vm/pmap.h>
46 
47 #include <machine/bus.h>
48 #include <machine/intr_machdep.h>
49 #include <machine/md_var.h>
50 #include <machine/rtas.h>
51 
52 #include <dev/ofw/ofw_bus.h>
53 #include <dev/ofw/ofw_bus_subr.h>
54 
55 #ifdef POWERNV
56 #include <powerpc/powernv/opal.h>
57 #endif
58 
59 #include "phyp-hvcall.h"
60 #include "pic_if.h"
61 
62 #define XICP_PRIORITY	5	/* Random non-zero number */
63 #define XICP_IPI	2
64 #define MAX_XICP_IRQS	(1<<24)	/* 24-bit XIRR field */
65 
66 static int	xicp_probe(device_t);
67 static int	xicp_attach(device_t);
68 static int	xics_probe(device_t);
69 static int	xics_attach(device_t);
70 
71 static void	xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv);
72 static void	xicp_dispatch(device_t, struct trapframe *);
73 static void	xicp_enable(device_t, u_int, u_int, void **priv);
74 static void	xicp_eoi(device_t, u_int, void *priv);
75 static void	xicp_ipi(device_t, u_int);
76 static void	xicp_mask(device_t, u_int, void *priv);
77 static void	xicp_unmask(device_t, u_int, void *priv);
78 
79 #ifdef POWERNV
80 extern void (*powernv_smp_ap_extra_init)(void);
81 static void	xicp_smp_cpu_startup(void);
82 #endif
83 
84 static device_method_t  xicp_methods[] = {
85 	/* Device interface */
86 	DEVMETHOD(device_probe,		xicp_probe),
87 	DEVMETHOD(device_attach,	xicp_attach),
88 
89 	/* PIC interface */
90 	DEVMETHOD(pic_bind,		xicp_bind),
91 	DEVMETHOD(pic_dispatch,		xicp_dispatch),
92 	DEVMETHOD(pic_enable,		xicp_enable),
93 	DEVMETHOD(pic_eoi,		xicp_eoi),
94 	DEVMETHOD(pic_ipi,		xicp_ipi),
95 	DEVMETHOD(pic_mask,		xicp_mask),
96 	DEVMETHOD(pic_unmask,		xicp_unmask),
97 
98 	DEVMETHOD_END
99 };
100 
101 static device_method_t  xics_methods[] = {
102 	/* Device interface */
103 	DEVMETHOD(device_probe,		xics_probe),
104 	DEVMETHOD(device_attach,	xics_attach),
105 
106 	DEVMETHOD_END
107 };
108 
109 struct xicp_intvec {
110 	int irq;
111 	int vector;
112 	int cpu;
113 };
114 
115 struct xicp_softc {
116 	struct mtx sc_mtx;
117 	struct resource *mem[MAXCPU];
118 
119 	int cpu_range[2];
120 
121 	int ibm_int_on;
122 	int ibm_int_off;
123 	int ibm_get_xive;
124 	int ibm_set_xive;
125 
126 	/* XXX: inefficient -- hash table? tree? */
127 	struct xicp_intvec intvecs[256];
128 	int nintvecs;
129 	int ipi_vec;
130 	bool xics_emu;
131 };
132 
133 static driver_t xicp_driver = {
134 	"xicp",
135 	xicp_methods,
136 	sizeof(struct xicp_softc)
137 };
138 
139 static driver_t xics_driver = {
140 	"xics",
141 	xics_methods,
142 	0
143 };
144 
145 #ifdef POWERNV
146 /* We can only pass physical addresses into OPAL.  Kernel stacks are in the KVA,
147  * not in the direct map, so we need to somehow extract the physical address.
148  * However, pmap_kextract() takes locks, which is forbidden in a critical region
149  * (which PIC_DISPATCH() operates in).  The kernel is mapped into the Direct
150  * Map (0xc000....), and the CPU implicitly drops the top two bits when doing
151  * real address by nature that the bus width is smaller than 64-bits.  Placing
152  * cpu_xirr into the DMAP lets us take advantage of this and avoids the
153  * pmap_kextract() that would otherwise be needed if using the stack variable.
154  */
155 static uint32_t cpu_xirr[MAXCPU];
156 #endif
157 
158 static devclass_t xicp_devclass;
159 static devclass_t xics_devclass;
160 
161 EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, xicp_devclass, 0, 0,
162     BUS_PASS_INTERRUPT-1);
163 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, xics_devclass, 0, 0,
164     BUS_PASS_INTERRUPT);
165 
166 #ifdef POWERNV
167 static struct resource *
168 xicp_mem_for_cpu(int cpu)
169 {
170 	device_t dev;
171 	struct xicp_softc *sc;
172 	int i;
173 
174 	for (i = 0; (dev = devclass_get_device(xicp_devclass, i)) != NULL; i++){
175 		sc = device_get_softc(dev);
176 		if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1])
177 			return (sc->mem[cpu - sc->cpu_range[0]]);
178 	}
179 
180 	return (NULL);
181 }
182 #endif
183 
184 static int
185 xicp_probe(device_t dev)
186 {
187 
188 	if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") &&
189 	    !ofw_bus_is_compatible(dev, "ibm,opal-intc"))
190 		return (ENXIO);
191 
192 	device_set_desc(dev, "External Interrupt Presentation Controller");
193 	return (BUS_PROBE_GENERIC);
194 }
195 
196 static int
197 xics_probe(device_t dev)
198 {
199 
200 	if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") &&
201 	    !ofw_bus_is_compatible(dev, "IBM,opal-xics"))
202 		return (ENXIO);
203 
204 	device_set_desc(dev, "External Interrupt Source Controller");
205 	return (BUS_PROBE_GENERIC);
206 }
207 
208 static int
209 xicp_attach(device_t dev)
210 {
211 	struct xicp_softc *sc = device_get_softc(dev);
212 	phandle_t phandle = ofw_bus_get_node(dev);
213 
214 	if (rtas_exists()) {
215 		sc->ibm_int_on = rtas_token_lookup("ibm,int-on");
216 		sc->ibm_int_off = rtas_token_lookup("ibm,int-off");
217 		sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive");
218 		sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive");
219 #ifdef POWERNV
220 	} else if (opal_check() == 0) {
221 		/* No init needed */
222 #endif
223 	} else {
224 		device_printf(dev, "Cannot attach without RTAS or OPAL\n");
225 		return (ENXIO);
226 	}
227 
228 	if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) {
229 		OF_getencprop(phandle, "ibm,interrupt-server-ranges",
230 		    sc->cpu_range, sizeof(sc->cpu_range));
231 		sc->cpu_range[1] += sc->cpu_range[0];
232 		device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0],
233 		    sc->cpu_range[1]-1);
234 #ifdef POWERNV
235 	} else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) {
236 			/*
237 			 * For now run POWER9 XIVE interrupt controller in XICS
238 			 * compatibility mode.
239 			 */
240 			sc->xics_emu = true;
241 			opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EMU);
242 #endif
243 	} else {
244 		sc->cpu_range[0] = 0;
245 		sc->cpu_range[1] = mp_ncpus;
246 	}
247 
248 #ifdef POWERNV
249 	if (mfmsr() & PSL_HV) {
250 		int i;
251 
252 		if (sc->xics_emu) {
253 			opal_call(OPAL_INT_SET_CPPR, 0xff);
254 			for (i = 0; i < mp_ncpus; i++) {
255 				opal_call(OPAL_INT_SET_MFRR,
256 				    pcpu_find(i)->pc_hwref, 0xff);
257 			}
258 		} else {
259 			for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) {
260 				sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
261 				    &i, RF_ACTIVE);
262 				if (sc->mem[i] == NULL) {
263 					device_printf(dev, "Could not alloc mem "
264 					    "resource %d\n", i);
265 					return (ENXIO);
266 				}
267 
268 				/* Unmask interrupts on all cores */
269 				bus_write_1(sc->mem[i], 4, 0xff);
270 				bus_write_1(sc->mem[i], 12, 0xff);
271 			}
272 		}
273 	}
274 #endif
275 
276 	mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF);
277 	sc->nintvecs = 0;
278 
279 	powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS,
280 	    1 /* Number of IPIs */, FALSE);
281 	root_pic = dev;
282 
283 #ifdef POWERNV
284 	if (sc->xics_emu)
285 		powernv_smp_ap_extra_init = xicp_smp_cpu_startup;
286 #endif
287 
288 	return (0);
289 }
290 
291 static int
292 xics_attach(device_t dev)
293 {
294 	phandle_t phandle = ofw_bus_get_node(dev);
295 
296 	/* The XICP (root PIC) will handle all our interrupts */
297 	powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
298 	    MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE);
299 
300 	return (0);
301 }
302 
303 static __inline struct xicp_intvec *
304 xicp_setup_priv(struct xicp_softc *sc, u_int irq, void **priv)
305 {
306 	if (*priv == NULL) {
307 		KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs),
308 			("Too many XICP interrupts"));
309 		mtx_lock(&sc->sc_mtx);
310 		*priv = &sc->intvecs[sc->nintvecs++];
311 		mtx_unlock(&sc->sc_mtx);
312 	}
313 
314 	return (*priv);
315 }
316 
317 /*
318  * PIC I/F methods.
319  */
320 
321 static void
322 xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
323 {
324 	struct xicp_softc *sc = device_get_softc(dev);
325 	struct xicp_intvec *iv;
326 	cell_t status, cpu;
327 	int ncpus, i, error = -1;
328 
329 	/* Ignore IPIs */
330 	if (irq == MAX_XICP_IRQS)
331 		return;
332 
333 	iv = xicp_setup_priv(sc, irq, priv);
334 
335 	/*
336 	 * This doesn't appear to actually support affinity groups, so pick a
337 	 * random CPU.
338 	 */
339 	ncpus = 0;
340 	CPU_FOREACH(cpu)
341 		if (CPU_ISSET(cpu, &cpumask)) ncpus++;
342 
343 	i = mftb() % ncpus;
344 	ncpus = 0;
345 	CPU_FOREACH(cpu) {
346 		if (!CPU_ISSET(cpu, &cpumask))
347 			continue;
348 		if (ncpus == i)
349 			break;
350 		ncpus++;
351 	}
352 
353 	cpu = pcpu_find(cpu)->pc_hwref;
354 	iv->cpu = cpu;
355 
356 	if (rtas_exists())
357 		error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
358 		    XICP_PRIORITY, &status);
359 #ifdef POWERNV
360 	else
361 		error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
362 #endif
363 
364 	if (error < 0)
365 		panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
366 }
367 
368 static void
369 xicp_dispatch(device_t dev, struct trapframe *tf)
370 {
371 	struct xicp_softc *sc;
372 	struct resource *regs = NULL;
373 	uint64_t xirr, junk;
374 	int i;
375 
376 	sc = device_get_softc(dev);
377 #ifdef POWERNV
378 	if ((mfmsr() & PSL_HV) && !sc->xics_emu) {
379 		regs = xicp_mem_for_cpu(PCPU_GET(hwref));
380 		KASSERT(regs != NULL,
381 		    ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref)));
382 	}
383 #endif
384 
385 	for (;;) {
386 		/* Return value in R4, use the PFT call */
387 		if (regs) {
388 			xirr = bus_read_4(regs, 4);
389 #ifdef POWERNV
390 		} else if (sc->xics_emu) {
391 			opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)],
392 			    false);
393 			xirr = cpu_xirr[PCPU_GET(cpuid)];
394 #endif
395 		} else {
396 			/* Return value in R4, use the PFT call */
397 			phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk);
398 		}
399 		xirr &= 0x00ffffff;
400 
401 		if (xirr == 0) /* No more pending interrupts? */
402 			break;
403 
404 		if (xirr == XICP_IPI) {		/* Magic number for IPIs */
405 			xirr = MAX_XICP_IRQS;	/* Map to FreeBSD magic */
406 
407 			/* Clear IPI */
408 			if (regs)
409 				bus_write_1(regs, 12, 0xff);
410 #ifdef POWERNV
411 			else if (sc->xics_emu)
412 				opal_call(OPAL_INT_SET_MFRR,
413 				    PCPU_GET(hwref), 0xff);
414 #endif
415 			else
416 				phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)),
417 				    0xff);
418 			i = sc->ipi_vec;
419 		} else {
420 
421 			/* XXX: super inefficient */
422 			for (i = 0; i < sc->nintvecs; i++) {
423 				if (sc->intvecs[i].irq == xirr)
424 					break;
425 			}
426 			KASSERT(i < sc->nintvecs, ("Unmapped XIRR"));
427 		}
428 
429 		powerpc_dispatch_intr(sc->intvecs[i].vector, tf);
430 	}
431 }
432 
433 static void
434 xicp_enable(device_t dev, u_int irq, u_int vector, void **priv)
435 {
436 	struct xicp_softc *sc;
437 	struct xicp_intvec *intr;
438 	cell_t status, cpu;
439 
440 	sc = device_get_softc(dev);
441 
442 	/* Bind to this CPU to start: distrib. ID is last entry in gserver# */
443 	cpu = PCPU_GET(hwref);
444 
445 	intr = xicp_setup_priv(sc, irq, priv);
446 
447 	intr->irq = irq;
448 	intr->vector = vector;
449 	intr->cpu = cpu;
450 	mb();
451 
452 	/* IPIs are also enabled.  Stash off the vector index */
453 	if (irq == MAX_XICP_IRQS) {
454 		sc->ipi_vec = intr - sc->intvecs;
455 		return;
456 	}
457 
458 	if (rtas_exists()) {
459 		rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
460 		    XICP_PRIORITY, &status);
461 		xicp_unmask(dev, irq, intr);
462 #ifdef POWERNV
463 	} else {
464 		status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
465 		/* Unmask implicit for OPAL */
466 
467 		if (status != 0)
468 			panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
469 			    cpu, status);
470 #endif
471 	}
472 }
473 
474 static void
475 xicp_eoi(device_t dev, u_int irq, void *priv)
476 {
477 #ifdef POWERNV
478 	struct xicp_softc *sc;
479 #endif
480 	uint64_t xirr;
481 
482 	if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */
483 		irq = XICP_IPI;
484 	xirr = irq | (0xff << 24);
485 
486 #ifdef POWERNV
487 	if (mfmsr() & PSL_HV) {
488 		sc = device_get_softc(dev);
489 		if (sc->xics_emu)
490 			opal_call(OPAL_INT_EOI, xirr);
491 		else
492 			bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr);
493 	} else
494 #endif
495 		phyp_hcall(H_EOI, xirr);
496 }
497 
498 static void
499 xicp_ipi(device_t dev, u_int cpu)
500 {
501 
502 #ifdef POWERNV
503 	struct xicp_softc *sc;
504 	cpu = pcpu_find(cpu)->pc_hwref;
505 
506 	if (mfmsr() & PSL_HV) {
507 		sc = device_get_softc(dev);
508 		if (sc->xics_emu) {
509 			int64_t rv;
510 			rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY);
511 			if (rv != 0)
512 			    device_printf(dev, "IPI SET_MFRR result: %ld\n", rv);
513 		} else
514 			bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY);
515 	} else
516 #endif
517 		phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY);
518 }
519 
520 static void
521 xicp_mask(device_t dev, u_int irq, void *priv)
522 {
523 	struct xicp_softc *sc = device_get_softc(dev);
524 	cell_t status;
525 
526 	if (irq == MAX_XICP_IRQS)
527 		return;
528 
529 	if (rtas_exists()) {
530 		rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status);
531 #ifdef POWERNV
532 	} else {
533 		struct xicp_intvec *ivec = priv;
534 
535 		KASSERT(ivec != NULL, ("Masking unconfigured interrupt"));
536 		opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, 0xff);
537 #endif
538 	}
539 }
540 
541 static void
542 xicp_unmask(device_t dev, u_int irq, void *priv)
543 {
544 	struct xicp_softc *sc = device_get_softc(dev);
545 	cell_t status;
546 
547 	if (irq == MAX_XICP_IRQS)
548 		return;
549 
550 	if (rtas_exists()) {
551 		rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status);
552 #ifdef POWERNV
553 	} else {
554 		struct xicp_intvec *ivec = priv;
555 
556 		KASSERT(ivec != NULL, ("Unmasking unconfigured interrupt"));
557 		opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, XICP_PRIORITY);
558 #endif
559 	}
560 }
561 
562 #ifdef POWERNV
563 /* This is only used on POWER9 systems with the XIVE's XICS emulation. */
564 static void
565 xicp_smp_cpu_startup(void)
566 {
567 	struct xicp_softc *sc;
568 
569 	if (mfmsr() & PSL_HV) {
570 		sc = device_get_softc(root_pic);
571 
572 		if (sc->xics_emu)
573 			opal_call(OPAL_INT_SET_CPPR, 0xff);
574 	}
575 }
576 #endif
577