1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright 2019 Justin Hibbits
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 #include "opt_platform.h"
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/module.h>
34 #include <sys/bus.h>
35 #include <sys/conf.h>
36 #include <sys/endian.h>
37 #include <sys/kernel.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/mutex.h>
41 #include <sys/smp.h>
42
43 #include <vm/vm.h>
44 #include <vm/pmap.h>
45
46 #include <machine/bus.h>
47 #include <machine/intr_machdep.h>
48 #include <machine/md_var.h>
49
50 #include <dev/ofw/ofw_bus.h>
51 #include <dev/ofw/ofw_bus_subr.h>
52
53 #ifdef POWERNV
54 #include <powerpc/powernv/opal.h>
55 #endif
56
57 #include "pic_if.h"
58
59 #define XIVE_PRIORITY 7 /* Random non-zero number */
60 #define MAX_XIVE_IRQS (1<<24) /* 24-bit XIRR field */
61
62 /* Registers */
63 #define XIVE_TM_QW1_OS 0x010 /* Guest OS registers */
64 #define XIVE_TM_QW2_HV_POOL 0x020 /* Hypervisor pool registers */
65 #define XIVE_TM_QW3_HV 0x030 /* Hypervisor registers */
66
67 #define XIVE_TM_NSR 0x00
68 #define XIVE_TM_CPPR 0x01
69 #define XIVE_TM_IPB 0x02
70 #define XIVE_TM_LSMFB 0x03
71 #define XIVE_TM_ACK_CNT 0x04
72 #define XIVE_TM_INC 0x05
73 #define XIVE_TM_AGE 0x06
74 #define XIVE_TM_PIPR 0x07
75
76 #define TM_WORD0 0x0
77 #define TM_WORD2 0x8
78 #define TM_QW2W2_VP 0x80000000
79
80 #define XIVE_TM_SPC_ACK 0x800
81 #define TM_QW3NSR_HE_SHIFT 14
82 #define TM_QW3_NSR_HE_NONE 0
83 #define TM_QW3_NSR_HE_POOL 1
84 #define TM_QW3_NSR_HE_PHYS 2
85 #define TM_QW3_NSR_HE_LSI 3
86 #define XIVE_TM_SPC_PULL_POOL_CTX 0x828
87
88 #define XIVE_IRQ_LOAD_EOI 0x000
89 #define XIVE_IRQ_STORE_EOI 0x400
90 #define XIVE_IRQ_PQ_00 0xc00
91 #define XIVE_IRQ_PQ_01 0xd00
92
93 #define XIVE_IRQ_VAL_P 0x02
94 #define XIVE_IRQ_VAL_Q 0x01
95
96 struct xive_softc;
97 struct xive_irq;
98
99 extern void (*powernv_smp_ap_extra_init)(void);
100
101 /* Private support */
102 static void xive_setup_cpu(void);
103 static void xive_smp_cpu_startup(void);
104 static void xive_init_irq(struct xive_irq *irqd, u_int irq);
105 static struct xive_irq *xive_configure_irq(u_int irq);
106 static int xive_provision_page(struct xive_softc *sc);
107
108 /* Interfaces */
109 static int xive_probe(device_t);
110 static int xive_attach(device_t);
111 static int xics_probe(device_t);
112 static int xics_attach(device_t);
113
114 static void xive_bind(device_t, u_int, cpuset_t, void **);
115 static void xive_dispatch(device_t, struct trapframe *);
116 static void xive_enable(device_t, u_int, u_int, void **);
117 static void xive_eoi(device_t, u_int, void *);
118 static void xive_ipi(device_t, u_int);
119 static void xive_mask(device_t, u_int, void *);
120 static void xive_unmask(device_t, u_int, void *);
121 static void xive_translate_code(device_t dev, u_int irq, int code,
122 enum intr_trigger *trig, enum intr_polarity *pol);
123
124 static device_method_t xive_methods[] = {
125 /* Device interface */
126 DEVMETHOD(device_probe, xive_probe),
127 DEVMETHOD(device_attach, xive_attach),
128
129 /* PIC interface */
130 DEVMETHOD(pic_bind, xive_bind),
131 DEVMETHOD(pic_dispatch, xive_dispatch),
132 DEVMETHOD(pic_enable, xive_enable),
133 DEVMETHOD(pic_eoi, xive_eoi),
134 DEVMETHOD(pic_ipi, xive_ipi),
135 DEVMETHOD(pic_mask, xive_mask),
136 DEVMETHOD(pic_unmask, xive_unmask),
137 DEVMETHOD(pic_translate_code, xive_translate_code),
138
139 DEVMETHOD_END
140 };
141
142 static device_method_t xics_methods[] = {
143 /* Device interface */
144 DEVMETHOD(device_probe, xics_probe),
145 DEVMETHOD(device_attach, xics_attach),
146
147 DEVMETHOD_END
148 };
149
150 struct xive_softc {
151 struct mtx sc_mtx;
152 struct resource *sc_mem;
153 vm_size_t sc_prov_page_size;
154 uint32_t sc_offset;
155 };
156
157 struct xive_queue {
158 uint32_t *q_page;
159 uint32_t *q_eoi_page;
160 uint32_t q_toggle;
161 uint32_t q_size;
162 uint32_t q_index;
163 uint32_t q_mask;
164 };
165
166 struct xive_irq {
167 uint32_t girq;
168 uint32_t lirq;
169 uint64_t vp;
170 uint64_t flags;
171 #define OPAL_XIVE_IRQ_SHIFT_BUG 0x00000008
172 #define OPAL_XIVE_IRQ_LSI 0x00000004
173 #define OPAL_XIVE_IRQ_STORE_EOI 0x00000002
174 #define OPAL_XIVE_IRQ_TRIGGER_PAGE 0x00000001
175 uint8_t prio;
176 vm_offset_t eoi_page;
177 vm_offset_t trig_page;
178 vm_size_t esb_size;
179 int chip;
180 };
181
182 struct xive_cpu {
183 uint64_t vp;
184 uint64_t flags;
185 struct xive_irq ipi_data;
186 struct xive_queue queue; /* We only use a single queue for now. */
187 uint64_t cam;
188 uint32_t chip;
189 };
190
191 static driver_t xive_driver = {
192 "xive",
193 xive_methods,
194 sizeof(struct xive_softc)
195 };
196
197 static driver_t xics_driver = {
198 "xivevc",
199 xics_methods,
200 0
201 };
202
203 EARLY_DRIVER_MODULE(xive, ofwbus, xive_driver, 0, 0, BUS_PASS_INTERRUPT - 1);
204 EARLY_DRIVER_MODULE(xivevc, ofwbus, xics_driver, 0, 0, BUS_PASS_INTERRUPT);
205
206 MALLOC_DEFINE(M_XIVE, "xive", "XIVE Memory");
207
208 DPCPU_DEFINE_STATIC(struct xive_cpu, xive_cpu_data);
209
210 static int xive_ipi_vector = -1;
211
212 /*
213 * XIVE Exploitation mode driver.
214 *
215 * The XIVE, present in the POWER9 CPU, can run in two modes: XICS emulation
216 * mode, and "Exploitation mode". XICS emulation mode is compatible with the
217 * POWER8 and earlier XICS interrupt controller, using OPAL calls to emulate
218 * hypervisor calls and memory accesses. Exploitation mode gives us raw access
219 * to the XIVE MMIO, improving performance significantly.
220 *
221 * The XIVE controller is a very bizarre interrupt controller. It uses queues
222 * in memory to pass interrupts around, and maps itself into 512GB of physical
223 * device address space, giving each interrupt in the system one or more pages
224 * of address space. An IRQ is tied to a virtual processor, which could be a
225 * physical CPU thread, or a guest CPU thread (LPAR running on a physical
226 * thread). Thus, the controller can route interrupts directly to guest OSes
227 * bypassing processing by the hypervisor, thereby improving performance of the
228 * guest OS.
229 *
230 * An IRQ, in addition to being tied to a virtual processor, has one or two
231 * page mappings: an EOI page, and an optional trigger page. The trigger page
232 * could be the same as the EOI page. Level-sensitive interrupts (LSIs) don't
233 * have a trigger page, as they're external interrupts controlled by physical
234 * lines. MSIs and IPIs have trigger pages. An IPI is really just another IRQ
235 * in the XIVE, which is triggered by software.
236 *
237 * An interesting behavior of the XIVE controller is that oftentimes the
238 * contents of an address location don't actually matter, but the direction of
239 * the action is the signifier (read vs write), and the address is significant.
240 * Hence, masking and unmasking an interrupt is done by reading different
241 * addresses in the EOI page, and triggering an interrupt consists of writing to
242 * the trigger page.
243 *
244 * Additionally, the MMIO region mapped is CPU-sensitive, just like the
245 * per-processor register space (private access) in OpenPIC. In order for a CPU
246 * to receive interrupts it must itself configure its CPPR (Current Processor
247 * Priority Register), it cannot be set by any other processor. This
248 * necessitates the xive_smp_cpu_startup() function.
249 *
250 * Queues are pages of memory, sized powers-of-two, that are shared with the
251 * XIVE. The XIVE writes into the queue with an alternating polarity bit, which
252 * flips when the queue wraps.
253 */
254
255 /*
256 * Offset-based read/write interfaces.
257 */
258 static uint16_t
xive_read_2(struct xive_softc * sc,bus_size_t offset)259 xive_read_2(struct xive_softc *sc, bus_size_t offset)
260 {
261
262 return (bus_read_2(sc->sc_mem, sc->sc_offset + offset));
263 }
264
265 static void
xive_write_1(struct xive_softc * sc,bus_size_t offset,uint8_t val)266 xive_write_1(struct xive_softc *sc, bus_size_t offset, uint8_t val)
267 {
268
269 bus_write_1(sc->sc_mem, sc->sc_offset + offset, val);
270 }
271
272 /* EOI and Trigger page access interfaces. */
273 static uint64_t
xive_read_mmap8(vm_offset_t addr)274 xive_read_mmap8(vm_offset_t addr)
275 {
276 return (*(volatile uint64_t *)addr);
277 }
278
279 static void
xive_write_mmap8(vm_offset_t addr,uint64_t val)280 xive_write_mmap8(vm_offset_t addr, uint64_t val)
281 {
282 *(uint64_t *)(addr) = val;
283 }
284
285 /* Device interfaces. */
286 static int
xive_probe(device_t dev)287 xive_probe(device_t dev)
288 {
289
290 if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-pe"))
291 return (ENXIO);
292
293 device_set_desc(dev, "External Interrupt Virtualization Engine");
294
295 /* Make sure we always win against the xicp driver. */
296 return (BUS_PROBE_DEFAULT);
297 }
298
299 static int
xics_probe(device_t dev)300 xics_probe(device_t dev)
301 {
302
303 if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-vc"))
304 return (ENXIO);
305
306 device_set_desc(dev, "External Interrupt Virtualization Engine Root");
307 return (BUS_PROBE_DEFAULT);
308 }
309
310 static int
xive_attach(device_t dev)311 xive_attach(device_t dev)
312 {
313 struct xive_softc *sc = device_get_softc(dev);
314 struct xive_cpu *xive_cpud;
315 phandle_t phandle = ofw_bus_get_node(dev);
316 int64_t vp_block;
317 int error;
318 int rid;
319 int i, order;
320 uint64_t vp_id;
321 int64_t ipi_irq;
322
323 opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EXP);
324
325 error = OF_getencprop(phandle, "ibm,xive-provision-page-size",
326 (pcell_t *)&sc->sc_prov_page_size, sizeof(sc->sc_prov_page_size));
327
328 rid = 1; /* Get the Hypervisor-level register set. */
329 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
330 &rid, RF_ACTIVE);
331 sc->sc_offset = XIVE_TM_QW3_HV;
332
333 mtx_init(&sc->sc_mtx, "XIVE", NULL, MTX_DEF);
334
335 /* Workaround for qemu single-thread powernv */
336 if (mp_maxid == 0)
337 order = 1;
338 else
339 order = fls(mp_maxid + (mp_maxid - 1)) - 1;
340
341 do {
342 vp_block = opal_call(OPAL_XIVE_ALLOCATE_VP_BLOCK, order);
343 if (vp_block == OPAL_BUSY)
344 DELAY(10);
345 else if (vp_block == OPAL_XIVE_PROVISIONING)
346 xive_provision_page(sc);
347 else
348 break;
349 } while (1);
350
351 if (vp_block < 0) {
352 device_printf(dev,
353 "Unable to allocate VP block. Opal error %d\n",
354 (int)vp_block);
355 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem);
356 return (ENXIO);
357 }
358
359 /*
360 * Set up the VPs. Try to do as much as we can in attach, to lessen
361 * what's needed at AP spawn time.
362 */
363 CPU_FOREACH(i) {
364 vp_id = pcpu_find(i)->pc_hwref;
365
366 xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data);
367 xive_cpud->vp = vp_id + vp_block;
368 opal_call(OPAL_XIVE_GET_VP_INFO, xive_cpud->vp, NULL,
369 vtophys(&xive_cpud->cam), NULL, vtophys(&xive_cpud->chip));
370
371 xive_cpud->cam = be64toh(xive_cpud->cam);
372 xive_cpud->chip = be64toh(xive_cpud->chip);
373
374 /* Allocate the queue page and populate the queue state data. */
375 xive_cpud->queue.q_page = contigmalloc(PAGE_SIZE, M_XIVE,
376 M_ZERO | M_WAITOK, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0);
377 xive_cpud->queue.q_size = 1 << PAGE_SHIFT;
378 xive_cpud->queue.q_mask =
379 ((xive_cpud->queue.q_size / sizeof(int)) - 1);
380 xive_cpud->queue.q_toggle = 0;
381 xive_cpud->queue.q_index = 0;
382 do {
383 error = opal_call(OPAL_XIVE_SET_VP_INFO, xive_cpud->vp,
384 OPAL_XIVE_VP_ENABLED, 0);
385 } while (error == OPAL_BUSY);
386 error = opal_call(OPAL_XIVE_SET_QUEUE_INFO, vp_id,
387 XIVE_PRIORITY, vtophys(xive_cpud->queue.q_page), PAGE_SHIFT,
388 OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED);
389
390 do {
391 ipi_irq = opal_call(OPAL_XIVE_ALLOCATE_IRQ,
392 xive_cpud->chip);
393 } while (ipi_irq == OPAL_BUSY);
394
395 if (ipi_irq < 0)
396 device_printf(root_pic,
397 "Failed allocating IPI. OPAL error %d\n",
398 (int)ipi_irq);
399 else {
400 xive_init_irq(&xive_cpud->ipi_data, ipi_irq);
401 xive_cpud->ipi_data.vp = vp_id;
402 xive_cpud->ipi_data.lirq = MAX_XIVE_IRQS;
403 opal_call(OPAL_XIVE_SET_IRQ_CONFIG, ipi_irq,
404 xive_cpud->ipi_data.vp, XIVE_PRIORITY,
405 MAX_XIVE_IRQS);
406 }
407 }
408
409 powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XIVE_IRQS,
410 1 /* Number of IPIs */, FALSE);
411 root_pic = dev;
412
413 xive_setup_cpu();
414 powernv_smp_ap_extra_init = xive_smp_cpu_startup;
415
416 return (0);
417 }
418
419 static int
xics_attach(device_t dev)420 xics_attach(device_t dev)
421 {
422 phandle_t phandle = ofw_bus_get_node(dev);
423
424 /* The XIVE (root PIC) will handle all our interrupts */
425 powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
426 MAX_XIVE_IRQS, 1 /* Number of IPIs */, FALSE);
427
428 return (0);
429 }
430
431 /*
432 * PIC I/F methods.
433 */
434
435 static void
xive_bind(device_t dev,u_int irq,cpuset_t cpumask,void ** priv)436 xive_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
437 {
438 struct xive_irq *irqd;
439 int cpu;
440 int ncpus, i, error;
441
442 if (*priv == NULL)
443 *priv = xive_configure_irq(irq);
444
445 irqd = *priv;
446
447 /*
448 * This doesn't appear to actually support affinity groups, so pick a
449 * random CPU.
450 */
451 ncpus = 0;
452 CPU_FOREACH(cpu)
453 if (CPU_ISSET(cpu, &cpumask)) ncpus++;
454
455 i = mftb() % ncpus;
456 ncpus = 0;
457 CPU_FOREACH(cpu) {
458 if (!CPU_ISSET(cpu, &cpumask))
459 continue;
460 if (ncpus == i)
461 break;
462 ncpus++;
463 }
464
465 opal_call(OPAL_XIVE_SYNC, OPAL_XIVE_SYNC_QUEUE, irq);
466
467 irqd->vp = pcpu_find(cpu)->pc_hwref;
468 error = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, irqd->vp,
469 XIVE_PRIORITY, irqd->lirq);
470
471 if (error < 0)
472 panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
473
474 xive_eoi(dev, irq, irqd);
475 }
476
477 /* Read the next entry in the queue page and update the index. */
478 static int
xive_read_eq(struct xive_queue * q)479 xive_read_eq(struct xive_queue *q)
480 {
481 uint32_t i = be32toh(q->q_page[q->q_index]);
482
483 /* Check validity, using current queue polarity. */
484 if ((i >> 31) == q->q_toggle)
485 return (0);
486
487 q->q_index = (q->q_index + 1) & q->q_mask;
488
489 if (q->q_index == 0)
490 q->q_toggle ^= 1;
491
492 return (i & 0x7fffffff);
493 }
494
495 static void
xive_dispatch(device_t dev,struct trapframe * tf)496 xive_dispatch(device_t dev, struct trapframe *tf)
497 {
498 struct xive_softc *sc;
499 struct xive_cpu *xive_cpud;
500 uint32_t vector;
501 uint16_t ack;
502 uint8_t cppr, he;
503
504 sc = device_get_softc(dev);
505
506 xive_cpud = DPCPU_PTR(xive_cpu_data);
507 for (;;) {
508 ack = xive_read_2(sc, XIVE_TM_SPC_ACK);
509 cppr = (ack & 0xff);
510
511 he = ack >> TM_QW3NSR_HE_SHIFT;
512
513 if (he == TM_QW3_NSR_HE_NONE)
514 break;
515
516 else if (__predict_false(he != TM_QW3_NSR_HE_PHYS)) {
517 /*
518 * We don't support TM_QW3_NSR_HE_POOL or
519 * TM_QW3_NSR_HE_LSI interrupts.
520 */
521 device_printf(dev,
522 "Unexpected interrupt he type: %d\n", he);
523 goto end;
524 }
525
526 xive_write_1(sc, XIVE_TM_CPPR, cppr);
527
528 for (;;) {
529 vector = xive_read_eq(&xive_cpud->queue);
530
531 if (vector == 0)
532 break;
533
534 if (vector == MAX_XIVE_IRQS)
535 vector = xive_ipi_vector;
536
537 powerpc_dispatch_intr(vector, tf);
538 }
539 }
540 end:
541 xive_write_1(sc, XIVE_TM_CPPR, 0xff);
542 }
543
544 static void
xive_enable(device_t dev,u_int irq,u_int vector,void ** priv)545 xive_enable(device_t dev, u_int irq, u_int vector, void **priv)
546 {
547 struct xive_irq *irqd;
548 cell_t status, cpu;
549
550 if (irq == MAX_XIVE_IRQS) {
551 if (xive_ipi_vector == -1)
552 xive_ipi_vector = vector;
553 return;
554 }
555 if (*priv == NULL)
556 *priv = xive_configure_irq(irq);
557
558 irqd = *priv;
559
560 /* Bind to this CPU to start */
561 cpu = PCPU_GET(hwref);
562 irqd->lirq = vector;
563
564 for (;;) {
565 status = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, cpu,
566 XIVE_PRIORITY, vector);
567 if (status != OPAL_BUSY)
568 break;
569 DELAY(10);
570 }
571
572 if (status != 0)
573 panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
574 cpu, status);
575
576 xive_unmask(dev, irq, *priv);
577 }
578
579 static void
xive_eoi(device_t dev,u_int irq,void * priv)580 xive_eoi(device_t dev, u_int irq, void *priv)
581 {
582 struct xive_irq *rirq;
583 struct xive_cpu *cpud;
584 uint8_t eoi_val;
585
586 if (irq == MAX_XIVE_IRQS) {
587 cpud = DPCPU_PTR(xive_cpu_data);
588 rirq = &cpud->ipi_data;
589 } else
590 rirq = priv;
591
592 if (rirq->flags & OPAL_XIVE_IRQ_STORE_EOI)
593 xive_write_mmap8(rirq->eoi_page + XIVE_IRQ_STORE_EOI, 0);
594 else if (rirq->flags & OPAL_XIVE_IRQ_LSI)
595 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_LOAD_EOI);
596 else {
597 eoi_val = xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
598 if ((eoi_val & XIVE_IRQ_VAL_Q) && rirq->trig_page != 0)
599 xive_write_mmap8(rirq->trig_page, 0);
600 }
601 }
602
603 static void
xive_ipi(device_t dev,u_int cpu)604 xive_ipi(device_t dev, u_int cpu)
605 {
606 struct xive_cpu *xive_cpud;
607
608 xive_cpud = DPCPU_ID_PTR(cpu, xive_cpu_data);
609
610 if (xive_cpud->ipi_data.trig_page == 0)
611 return;
612 xive_write_mmap8(xive_cpud->ipi_data.trig_page, 0);
613 }
614
615 static void
xive_mask(device_t dev,u_int irq,void * priv)616 xive_mask(device_t dev, u_int irq, void *priv)
617 {
618 struct xive_irq *rirq;
619
620 /* Never mask IPIs */
621 if (irq == MAX_XIVE_IRQS)
622 return;
623
624 rirq = priv;
625
626 if (!(rirq->flags & OPAL_XIVE_IRQ_LSI))
627 return;
628 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_01);
629 }
630
631 static void
xive_unmask(device_t dev,u_int irq,void * priv)632 xive_unmask(device_t dev, u_int irq, void *priv)
633 {
634 struct xive_irq *rirq;
635
636 rirq = priv;
637
638 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00);
639 }
640
641 static void
xive_translate_code(device_t dev,u_int irq,int code,enum intr_trigger * trig,enum intr_polarity * pol)642 xive_translate_code(device_t dev, u_int irq, int code,
643 enum intr_trigger *trig, enum intr_polarity *pol)
644 {
645 switch (code) {
646 case 0:
647 /* L to H edge */
648 *trig = INTR_TRIGGER_EDGE;
649 *pol = INTR_POLARITY_HIGH;
650 break;
651 case 1:
652 /* Active L level */
653 *trig = INTR_TRIGGER_LEVEL;
654 *pol = INTR_POLARITY_LOW;
655 break;
656 default:
657 *trig = INTR_TRIGGER_CONFORM;
658 *pol = INTR_POLARITY_CONFORM;
659 }
660 }
661
662 /* Private functions. */
663 /*
664 * Setup the current CPU. Called by the BSP at driver attachment, and by each
665 * AP at wakeup (via xive_smp_cpu_startup()).
666 */
667 static void
xive_setup_cpu(void)668 xive_setup_cpu(void)
669 {
670 struct xive_softc *sc;
671 struct xive_cpu *cpup;
672 uint32_t val;
673
674 cpup = DPCPU_PTR(xive_cpu_data);
675
676 sc = device_get_softc(root_pic);
677
678 val = bus_read_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2);
679 if (val & TM_QW2W2_VP)
680 bus_read_8(sc->sc_mem, XIVE_TM_SPC_PULL_POOL_CTX);
681
682 bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD0, 0xff);
683 bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2,
684 TM_QW2W2_VP | cpup->cam);
685
686 xive_unmask(root_pic, cpup->ipi_data.girq, &cpup->ipi_data);
687 xive_write_1(sc, XIVE_TM_CPPR, 0xff);
688 }
689
690 /* Populate an IRQ structure, mapping the EOI and trigger pages. */
691 static void
xive_init_irq(struct xive_irq * irqd,u_int irq)692 xive_init_irq(struct xive_irq *irqd, u_int irq)
693 {
694 uint64_t eoi_phys, trig_phys;
695 uint32_t esb_shift;
696
697 opal_call(OPAL_XIVE_GET_IRQ_INFO, irq,
698 vtophys(&irqd->flags), vtophys(&eoi_phys),
699 vtophys(&trig_phys), vtophys(&esb_shift),
700 vtophys(&irqd->chip));
701
702 irqd->flags = be64toh(irqd->flags);
703 eoi_phys = be64toh(eoi_phys);
704 trig_phys = be64toh(trig_phys);
705 esb_shift = be32toh(esb_shift);
706 irqd->chip = be32toh(irqd->chip);
707
708 irqd->girq = irq;
709 irqd->esb_size = 1 << esb_shift;
710 irqd->eoi_page = (vm_offset_t)pmap_mapdev(eoi_phys, irqd->esb_size);
711
712 if (eoi_phys == trig_phys)
713 irqd->trig_page = irqd->eoi_page;
714 else if (trig_phys != 0)
715 irqd->trig_page = (vm_offset_t)pmap_mapdev(trig_phys,
716 irqd->esb_size);
717 else
718 irqd->trig_page = 0;
719
720 opal_call(OPAL_XIVE_GET_IRQ_CONFIG, irq, vtophys(&irqd->vp),
721 vtophys(&irqd->prio), vtophys(&irqd->lirq));
722
723 irqd->vp = be64toh(irqd->vp);
724 irqd->prio = be64toh(irqd->prio);
725 irqd->lirq = be32toh(irqd->lirq);
726 }
727
728 /* Allocate an IRQ struct before populating it. */
729 static struct xive_irq *
xive_configure_irq(u_int irq)730 xive_configure_irq(u_int irq)
731 {
732 struct xive_irq *irqd;
733
734 irqd = malloc(sizeof(struct xive_irq), M_XIVE, M_WAITOK);
735
736 xive_init_irq(irqd, irq);
737
738 return (irqd);
739 }
740
741 /*
742 * Part of the OPAL API. OPAL_XIVE_ALLOCATE_VP_BLOCK might require more pages,
743 * provisioned through this call.
744 */
745 static int
xive_provision_page(struct xive_softc * sc)746 xive_provision_page(struct xive_softc *sc)
747 {
748 void *prov_page;
749 int error;
750
751 do {
752 prov_page = contigmalloc(sc->sc_prov_page_size, M_XIVE, 0,
753 0, BUS_SPACE_MAXADDR,
754 sc->sc_prov_page_size, sc->sc_prov_page_size);
755
756 error = opal_call(OPAL_XIVE_DONATE_PAGE, -1,
757 vtophys(prov_page));
758 } while (error == OPAL_XIVE_PROVISIONING);
759
760 return (0);
761 }
762
763 /* The XIVE_TM_CPPR register must be set by each thread */
764 static void
xive_smp_cpu_startup(void)765 xive_smp_cpu_startup(void)
766 {
767
768 xive_setup_cpu();
769 }
770