xref: /freebsd/sys/powerpc/powernv/opal_pci.c (revision 5bf5ca772c6de2d53344a78cf461447cc322ccea)
1 /*-
2  * Copyright (c) 2015-2016 Nathan Whitehorn
3  * Copyright (c) 2017-2018 Semihalf
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/module.h>
33 #include <sys/bus.h>
34 #include <sys/conf.h>
35 #include <sys/kernel.h>
36 #include <sys/pciio.h>
37 #include <sys/endian.h>
38 #include <sys/rman.h>
39 #include <sys/vmem.h>
40 
41 #include <dev/ofw/openfirm.h>
42 #include <dev/ofw/ofw_pci.h>
43 #include <dev/ofw/ofw_bus.h>
44 #include <dev/ofw/ofw_bus_subr.h>
45 #include <dev/ofw/ofwpci.h>
46 
47 #include <dev/pci/pcivar.h>
48 #include <dev/pci/pcireg.h>
49 
50 #include <machine/bus.h>
51 #include <machine/intr_machdep.h>
52 #include <machine/md_var.h>
53 
54 #include <vm/vm.h>
55 #include <vm/pmap.h>
56 
57 #include "pcib_if.h"
58 #include "pic_if.h"
59 #include "iommu_if.h"
60 #include "opal.h"
61 
62 #define	OPAL_PCI_TCE_MAX_ENTRIES	(1024*1024UL)
63 #define	OPAL_PCI_TCE_SEG_SIZE		(16*1024*1024UL)
64 #define	OPAL_PCI_TCE_R			(1UL << 0)
65 #define	OPAL_PCI_TCE_W			(1UL << 1)
66 #define	PHB3_TCE_KILL_INVAL_ALL		(1UL << 63)
67 
68 /*
69  * Device interface.
70  */
71 static int		opalpci_probe(device_t);
72 static int		opalpci_attach(device_t);
73 
74 /*
75  * pcib interface.
76  */
77 static uint32_t		opalpci_read_config(device_t, u_int, u_int, u_int,
78 			    u_int, int);
79 static void		opalpci_write_config(device_t, u_int, u_int, u_int,
80 			    u_int, u_int32_t, int);
81 static int		opalpci_alloc_msi(device_t dev, device_t child,
82 			    int count, int maxcount, int *irqs);
83 static int		opalpci_release_msi(device_t dev, device_t child,
84 			    int count, int *irqs);
85 static int		opalpci_alloc_msix(device_t dev, device_t child,
86 			    int *irq);
87 static int		opalpci_release_msix(device_t dev, device_t child,
88 			    int irq);
89 static int		opalpci_map_msi(device_t dev, device_t child,
90 			    int irq, uint64_t *addr, uint32_t *data);
91 static int opalpci_route_interrupt(device_t bus, device_t dev, int pin);
92 
93 /*
94  * MSI PIC interface.
95  */
96 static void opalpic_pic_enable(device_t dev, u_int irq, u_int vector);
97 static void opalpic_pic_eoi(device_t dev, u_int irq);
98 static void opalpic_pic_mask(device_t dev, u_int irq);
99 static void opalpic_pic_unmask(device_t dev, u_int irq);
100 
101 /*
102  * Commands
103  */
104 #define	OPAL_M32_WINDOW_TYPE		1
105 #define	OPAL_M64_WINDOW_TYPE		2
106 #define	OPAL_IO_WINDOW_TYPE		3
107 
108 #define	OPAL_RESET_PHB_COMPLETE		1
109 #define	OPAL_RESET_PCI_IODA_TABLE	6
110 
111 #define	OPAL_DISABLE_M64		0
112 #define	OPAL_ENABLE_M64_SPLIT		1
113 #define	OPAL_ENABLE_M64_NON_SPLIT	2
114 
115 #define	OPAL_EEH_ACTION_CLEAR_FREEZE_MMIO	1
116 #define	OPAL_EEH_ACTION_CLEAR_FREEZE_DMA	2
117 #define	OPAL_EEH_ACTION_CLEAR_FREEZE_ALL	3
118 
119 /*
120  * Constants
121  */
122 #define OPAL_PCI_DEFAULT_PE			1
123 
124 /*
125  * Driver methods.
126  */
127 static device_method_t	opalpci_methods[] = {
128 	/* Device interface */
129 	DEVMETHOD(device_probe,		opalpci_probe),
130 	DEVMETHOD(device_attach,	opalpci_attach),
131 
132 	/* pcib interface */
133 	DEVMETHOD(pcib_read_config,	opalpci_read_config),
134 	DEVMETHOD(pcib_write_config,	opalpci_write_config),
135 
136 	DEVMETHOD(pcib_alloc_msi,	opalpci_alloc_msi),
137 	DEVMETHOD(pcib_release_msi,	opalpci_release_msi),
138 	DEVMETHOD(pcib_alloc_msix,	opalpci_alloc_msix),
139 	DEVMETHOD(pcib_release_msix,	opalpci_release_msix),
140 	DEVMETHOD(pcib_map_msi,		opalpci_map_msi),
141 	DEVMETHOD(pcib_route_interrupt,	opalpci_route_interrupt),
142 
143 	/* PIC interface for MSIs */
144 	DEVMETHOD(pic_enable,		opalpic_pic_enable),
145 	DEVMETHOD(pic_eoi,		opalpic_pic_eoi),
146 	DEVMETHOD(pic_mask,		opalpic_pic_mask),
147 	DEVMETHOD(pic_unmask,		opalpic_pic_unmask),
148 
149 	DEVMETHOD_END
150 };
151 
152 struct opalpci_softc {
153 	struct ofw_pci_softc ofw_sc;
154 	uint64_t phb_id;
155 	vmem_t *msi_vmem;
156 	int msi_base;		/* Base XIVE number */
157 	int base_msi_irq;	/* Base IRQ assigned by FreeBSD to this PIC */
158 	uint64_t *tce;		/* TCE table for 1:1 mapping */
159 	struct resource *r_reg;
160 };
161 
162 static devclass_t	opalpci_devclass;
163 DEFINE_CLASS_1(pcib, opalpci_driver, opalpci_methods,
164     sizeof(struct opalpci_softc), ofw_pci_driver);
165 EARLY_DRIVER_MODULE(opalpci, ofwbus, opalpci_driver, opalpci_devclass, 0, 0,
166     BUS_PASS_BUS);
167 
168 static int
169 opalpci_probe(device_t dev)
170 {
171 	const char	*type;
172 
173 	if (opal_check() != 0)
174 		return (ENXIO);
175 
176 	type = ofw_bus_get_type(dev);
177 
178 	if (type == NULL || (strcmp(type, "pci") != 0 &&
179 	    strcmp(type, "pciex") != 0))
180 		return (ENXIO);
181 
182 	if (!OF_hasprop(ofw_bus_get_node(dev), "ibm,opal-phbid"))
183 		return (ENXIO);
184 
185 	device_set_desc(dev, "OPAL Host-PCI bridge");
186 	return (BUS_PROBE_GENERIC);
187 }
188 
189 static void
190 pci_phb3_tce_invalidate_entire(struct opalpci_softc *sc)
191 {
192 
193 	mb();
194 	bus_write_8(sc->r_reg, 0x210, PHB3_TCE_KILL_INVAL_ALL);
195 	mb();
196 }
197 
198 static int
199 opalpci_attach(device_t dev)
200 {
201 	struct opalpci_softc *sc;
202 	cell_t id[2], m64window[6], npe;
203 	int i, err;
204 	uint64_t maxmem;
205 	uint64_t entries;
206 	int rid;
207 
208 	sc = device_get_softc(dev);
209 
210 	switch (OF_getproplen(ofw_bus_get_node(dev), "ibm,opal-phbid")) {
211 	case 8:
212 		OF_getencprop(ofw_bus_get_node(dev), "ibm,opal-phbid", id, 8);
213 		sc->phb_id = ((uint64_t)id[0] << 32) | id[1];
214 		break;
215 	case 4:
216 		OF_getencprop(ofw_bus_get_node(dev), "ibm,opal-phbid", id, 4);
217 		sc->phb_id = id[0];
218 		break;
219 	default:
220 		device_printf(dev, "PHB ID property had wrong length (%zd)\n",
221 		    OF_getproplen(ofw_bus_get_node(dev), "ibm,opal-phbid"));
222 		return (ENXIO);
223 	}
224 
225 	if (bootverbose)
226 		device_printf(dev, "OPAL ID %#lx\n", sc->phb_id);
227 
228 	rid = 0;
229 	sc->r_reg = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
230 	    &rid, RF_ACTIVE | RF_SHAREABLE);
231 	if (sc->r_reg == NULL) {
232 		device_printf(dev, "Failed to allocate PHB[%jd] registers\n",
233 		    (uintmax_t)sc->phb_id);
234 		return (ENXIO);
235 	}
236 
237 	/*
238 	 * Reset PCI IODA table
239 	 */
240 	err = opal_call(OPAL_PCI_RESET, sc->phb_id, OPAL_RESET_PCI_IODA_TABLE,
241 	    1);
242 	if (err != 0) {
243 		device_printf(dev, "IODA table reset failed: %d\n", err);
244 		return (ENXIO);
245 	}
246 	while ((err = opal_call(OPAL_PCI_POLL, sc->phb_id)) > 0)
247 		DELAY(1000*(err + 1)); /* Returns expected delay in ms */
248 	if (err < 0) {
249 		device_printf(dev, "WARNING: PHB IODA reset poll failed: %d\n", err);
250 	}
251 
252 	/*
253 	 * Map all devices on the bus to partitionable endpoint one until
254 	 * such time as we start wanting to do things like bhyve.
255 	 */
256 	err = opal_call(OPAL_PCI_SET_PE, sc->phb_id, OPAL_PCI_DEFAULT_PE,
257 	    0, 0, 0, 0, /* All devices */
258 	    OPAL_MAP_PE);
259 	if (err != 0) {
260 		device_printf(dev, "PE mapping failed: %d\n", err);
261 		return (ENXIO);
262 	}
263 
264 	/*
265 	 * Turn on MMIO, mapped to PE 1
266 	 */
267 	if (OF_getencprop(ofw_bus_get_node(dev), "ibm,opal-num-pes", &npe, 4)
268 	    != 4)
269 		npe = 1;
270 	for (i = 0; i < npe; i++) {
271 		err = opal_call(OPAL_PCI_MAP_PE_MMIO_WINDOW, sc->phb_id,
272 		    OPAL_PCI_DEFAULT_PE, OPAL_M32_WINDOW_TYPE, 0, i);
273 		if (err != 0)
274 			device_printf(dev, "MMIO %d map failed: %d\n", i, err);
275 	}
276 
277 	/* XXX: multiple M64 windows? */
278 	if (OF_getencprop(ofw_bus_get_node(dev), "ibm,opal-m64-window",
279 	    m64window, sizeof(m64window)) == sizeof(m64window)) {
280 		opal_call(OPAL_PCI_PHB_MMIO_ENABLE, sc->phb_id,
281 		    OPAL_M64_WINDOW_TYPE, 0, 0);
282 		opal_call(OPAL_PCI_SET_PHB_MEM_WINDOW, sc->phb_id,
283 		    OPAL_M64_WINDOW_TYPE, 0 /* index */,
284 		    ((uint64_t)m64window[2] << 32) | m64window[3], 0,
285 		    ((uint64_t)m64window[4] << 32) | m64window[5]);
286 		opal_call(OPAL_PCI_MAP_PE_MMIO_WINDOW, sc->phb_id,
287 		    OPAL_PCI_DEFAULT_PE, OPAL_M64_WINDOW_TYPE,
288 		    0 /* index */, 0);
289 		opal_call(OPAL_PCI_PHB_MMIO_ENABLE, sc->phb_id,
290 		    OPAL_M64_WINDOW_TYPE, 0, OPAL_ENABLE_M64_NON_SPLIT);
291 	}
292 
293 	/*
294 	 * Enable IOMMU for PE1 - map everything 1:1 using
295 	 * segments of OPAL_PCI_TCE_SEG_SIZE size
296 	 */
297 	maxmem = roundup2(powerpc_ptob(Maxmem), OPAL_PCI_TCE_SEG_SIZE);
298 	entries = maxmem / OPAL_PCI_TCE_SEG_SIZE;
299 	if (entries > OPAL_PCI_TCE_MAX_ENTRIES)
300 		panic("POWERNV supports only %jdGB of memory space\n",
301 		    (uintmax_t)((OPAL_PCI_TCE_MAX_ENTRIES * OPAL_PCI_TCE_SEG_SIZE) >> 30));
302 	if (bootverbose)
303 		device_printf(dev, "Mapping 0-%#jx for DMA\n", (uintmax_t)maxmem);
304 	sc->tce = contigmalloc(OPAL_PCI_TCE_MAX_ENTRIES * sizeof(uint64_t),
305 	    M_DEVBUF, M_NOWAIT | M_ZERO, 0,
306 	    BUS_SPACE_MAXADDR_32BIT, OPAL_PCI_TCE_SEG_SIZE, 0);
307 	if (sc->tce == NULL)
308 		panic("Failed to allocate TCE memory for PHB %jd\n",
309 		    (uintmax_t)sc->phb_id);
310 
311 	for (i = 0; i < entries; i++)
312 		sc->tce[i] = (i * OPAL_PCI_TCE_SEG_SIZE) | OPAL_PCI_TCE_R | OPAL_PCI_TCE_W;
313 
314 	/* Map TCE for every PE. It seems necessary for Power8 */
315 	for (i = 0; i < npe; i++) {
316 		err = opal_call(OPAL_PCI_MAP_PE_DMA_WINDOW, sc->phb_id,
317 		    i, (i << 1),
318 		    1, pmap_kextract((uint64_t)&sc->tce[0]),
319 		    OPAL_PCI_TCE_MAX_ENTRIES * sizeof(uint64_t), OPAL_PCI_TCE_SEG_SIZE);
320 		if (err != 0) {
321 			device_printf(dev, "DMA IOMMU mapping failed: %d\n", err);
322 			return (ENXIO);
323 		}
324 
325 		err = opal_call(OPAL_PCI_MAP_PE_DMA_WINDOW_REAL, sc->phb_id,
326 		    i, (i << 1) + 1,
327 		    (1UL << 59), maxmem);
328 		if (err != 0) {
329 			device_printf(dev, "DMA 64b bypass mapping failed: %d\n", err);
330 			return (ENXIO);
331 		}
332 	}
333 
334 	/*
335 	 * Invalidate all previous TCE entries.
336 	 *
337 	 * TODO: add support for other PHBs than PHB3
338 	 */
339 	pci_phb3_tce_invalidate_entire(sc);
340 
341 	/*
342 	 * Get MSI properties
343 	 */
344 	sc->msi_vmem = NULL;
345 	if (OF_getproplen(ofw_bus_get_node(dev), "ibm,opal-msi-ranges") > 0) {
346 		cell_t msi_ranges[2];
347 		OF_getencprop(ofw_bus_get_node(dev), "ibm,opal-msi-ranges",
348 		    msi_ranges, sizeof(msi_ranges));
349 		sc->msi_base = msi_ranges[0];
350 
351 		sc->msi_vmem = vmem_create("OPAL MSI", msi_ranges[0],
352 		    msi_ranges[1], 1, 16, M_BESTFIT | M_WAITOK);
353 
354 		sc->base_msi_irq = powerpc_register_pic(dev,
355 		    OF_xref_from_node(ofw_bus_get_node(dev)),
356 		    msi_ranges[0] + msi_ranges[1], 0, FALSE);
357 
358 		if (bootverbose)
359 			device_printf(dev, "Supports %d MSIs starting at %d\n",
360 			    msi_ranges[1], msi_ranges[0]);
361 	}
362 
363 	/*
364 	 * General OFW PCI attach
365 	 */
366 	err = ofw_pci_init(dev);
367 	if (err != 0)
368 		return (err);
369 
370 	/*
371 	 * Unfreeze non-config-space PCI operations. Let this fail silently
372 	 * if e.g. there is no current freeze.
373 	 */
374 	opal_call(OPAL_PCI_EEH_FREEZE_CLEAR, sc->phb_id, OPAL_PCI_DEFAULT_PE,
375 	    OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
376 
377 	/*
378 	 * OPAL stores 64-bit BARs in a special property rather than "ranges"
379 	 */
380 	if (OF_getencprop(ofw_bus_get_node(dev), "ibm,opal-m64-window",
381 	    m64window, sizeof(m64window)) == sizeof(m64window)) {
382 		struct ofw_pci_range *rp;
383 
384 		sc->ofw_sc.sc_nrange++;
385 		sc->ofw_sc.sc_range = realloc(sc->ofw_sc.sc_range,
386 		    sc->ofw_sc.sc_nrange * sizeof(sc->ofw_sc.sc_range[0]),
387 		    M_DEVBUF, M_WAITOK);
388 		rp = &sc->ofw_sc.sc_range[sc->ofw_sc.sc_nrange-1];
389 		rp->pci_hi = OFW_PCI_PHYS_HI_SPACE_MEM64 |
390 		    OFW_PCI_PHYS_HI_PREFETCHABLE;
391 		rp->pci = ((uint64_t)m64window[0] << 32) | m64window[1];
392 		rp->host = ((uint64_t)m64window[2] << 32) | m64window[3];
393 		rp->size = ((uint64_t)m64window[4] << 32) | m64window[5];
394 		rman_manage_region(&sc->ofw_sc.sc_mem_rman, rp->pci,
395 		   rp->pci + rp->size - 1);
396 	}
397 
398 	return (ofw_pci_attach(dev));
399 }
400 
401 static uint32_t
402 opalpci_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
403     int width)
404 {
405 	struct opalpci_softc *sc;
406 	uint64_t config_addr;
407 	uint8_t byte;
408 	uint16_t half;
409 	uint32_t word;
410 	int error;
411 
412 	sc = device_get_softc(dev);
413 
414 	config_addr = (bus << 8) | ((slot & 0x1f) << 3) | (func & 0x7);
415 
416 	switch (width) {
417 	case 1:
418 		error = opal_call(OPAL_PCI_CONFIG_READ_BYTE, sc->phb_id,
419 		    config_addr, reg, vtophys(&byte));
420 		word = byte;
421 		break;
422 	case 2:
423 		error = opal_call(OPAL_PCI_CONFIG_READ_HALF_WORD, sc->phb_id,
424 		    config_addr, reg, vtophys(&half));
425 		word = half;
426 		break;
427 	case 4:
428 		error = opal_call(OPAL_PCI_CONFIG_READ_WORD, sc->phb_id,
429 		    config_addr, reg, vtophys(&word));
430 		break;
431 	default:
432 		error = OPAL_SUCCESS;
433 		word = 0xffffffff;
434 	}
435 
436 	/*
437 	 * Poking config state for non-existant devices can make
438 	 * the host bridge hang up. Clear any errors.
439 	 *
440 	 * XXX: Make this conditional on the existence of a freeze
441 	 */
442 	opal_call(OPAL_PCI_EEH_FREEZE_CLEAR, sc->phb_id, OPAL_PCI_DEFAULT_PE,
443 	    OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
444 
445 	if (error != OPAL_SUCCESS)
446 		word = 0xffffffff;
447 
448 	return (word);
449 }
450 
451 static void
452 opalpci_write_config(device_t dev, u_int bus, u_int slot, u_int func,
453     u_int reg, uint32_t val, int width)
454 {
455 	struct opalpci_softc *sc;
456 	uint64_t config_addr;
457 	int error = OPAL_SUCCESS;
458 
459 	sc = device_get_softc(dev);
460 
461 	config_addr = (bus << 8) | ((slot & 0x1f) << 3) | (func & 0x7);
462 
463 	switch (width) {
464 	case 1:
465 		error = opal_call(OPAL_PCI_CONFIG_WRITE_BYTE, sc->phb_id,
466 		    config_addr, reg, val);
467 		break;
468 	case 2:
469 		error = opal_call(OPAL_PCI_CONFIG_WRITE_HALF_WORD, sc->phb_id,
470 		    config_addr, reg, val);
471 		break;
472 	case 4:
473 		error = opal_call(OPAL_PCI_CONFIG_WRITE_WORD, sc->phb_id,
474 		    config_addr, reg, val);
475 		break;
476 	}
477 
478 	if (error != OPAL_SUCCESS) {
479 		/*
480 		 * Poking config state for non-existant devices can make
481 		 * the host bridge hang up. Clear any errors.
482 		 */
483 		opal_call(OPAL_PCI_EEH_FREEZE_CLEAR, sc->phb_id,
484 		    OPAL_PCI_DEFAULT_PE, OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
485 	}
486 }
487 
488 static int
489 opalpci_route_interrupt(device_t bus, device_t dev, int pin)
490 {
491 
492 	return (pin);
493 }
494 
495 static int
496 opalpci_alloc_msi(device_t dev, device_t child, int count, int maxcount,
497     int *irqs)
498 {
499 	struct opalpci_softc *sc;
500 	vmem_addr_t start;
501 	phandle_t xref;
502 	int err, i;
503 
504 	sc = device_get_softc(dev);
505 	if (sc->msi_vmem == NULL)
506 		return (ENODEV);
507 
508 	err = vmem_xalloc(sc->msi_vmem, count, powerof2(count), 0, 0,
509 	    VMEM_ADDR_MIN, VMEM_ADDR_MAX, M_BESTFIT | M_WAITOK, &start);
510 
511 	if (err)
512 		return (err);
513 
514 	xref = OF_xref_from_node(ofw_bus_get_node(dev));
515 	for (i = 0; i < count; i++)
516 		irqs[i] = MAP_IRQ(xref, start + i);
517 
518 	return (0);
519 }
520 
521 static int
522 opalpci_release_msi(device_t dev, device_t child, int count, int *irqs)
523 {
524 	struct opalpci_softc *sc;
525 
526 	sc = device_get_softc(dev);
527 	if (sc->msi_vmem == NULL)
528 		return (ENODEV);
529 
530 	vmem_xfree(sc->msi_vmem, irqs[0] - sc->base_msi_irq, count);
531 	return (0);
532 }
533 
534 static int
535 opalpci_alloc_msix(device_t dev, device_t child, int *irq)
536 {
537 	return (opalpci_alloc_msi(dev, child, 1, 1, irq));
538 }
539 
540 static int
541 opalpci_release_msix(device_t dev, device_t child, int irq)
542 {
543 	return (opalpci_release_msi(dev, child, 1, &irq));
544 }
545 
546 static int
547 opalpci_map_msi(device_t dev, device_t child, int irq, uint64_t *addr,
548     uint32_t *data)
549 {
550 	struct opalpci_softc *sc;
551 	struct pci_devinfo *dinfo;
552 	int err, xive;
553 
554 	sc = device_get_softc(dev);
555 	if (sc->msi_vmem == NULL)
556 		return (ENODEV);
557 
558 	xive = irq - sc->base_msi_irq - sc->msi_base;
559 	opal_call(OPAL_PCI_SET_XIVE_PE, sc->phb_id, OPAL_PCI_DEFAULT_PE, xive);
560 
561 	dinfo = device_get_ivars(child);
562 	if (dinfo->cfg.msi.msi_alloc > 0 &&
563 	    (dinfo->cfg.msi.msi_ctrl & PCIM_MSICTRL_64BIT) == 0) {
564 		uint32_t msi32;
565 		err = opal_call(OPAL_GET_MSI_32, sc->phb_id,
566 		    OPAL_PCI_DEFAULT_PE, xive, 1, vtophys(&msi32),
567 		    vtophys(data));
568 		*addr = be32toh(msi32);
569 	} else {
570 		err = opal_call(OPAL_GET_MSI_64, sc->phb_id,
571 		    OPAL_PCI_DEFAULT_PE, xive, 1, vtophys(addr), vtophys(data));
572 		*addr = be64toh(*addr);
573 	}
574 	*data = be32toh(*data);
575 
576 	if (bootverbose && err != 0)
577 		device_printf(child, "OPAL MSI mapping error: %d\n", err);
578 
579 	return ((err == 0) ? 0 : ENXIO);
580 }
581 
582 static void
583 opalpic_pic_enable(device_t dev, u_int irq, u_int vector)
584 {
585 	PIC_ENABLE(root_pic, irq, vector);
586 }
587 
588 static void opalpic_pic_eoi(device_t dev, u_int irq)
589 {
590 	struct opalpci_softc *sc;
591 
592 	sc = device_get_softc(dev);
593 	opal_call(OPAL_PCI_MSI_EOI, sc->phb_id, irq);
594 
595 	PIC_EOI(root_pic, irq);
596 }
597 
598 static void opalpic_pic_mask(device_t dev, u_int irq)
599 {
600 	PIC_MASK(root_pic, irq);
601 }
602 
603 static void opalpic_pic_unmask(device_t dev, u_int irq)
604 {
605 	struct opalpci_softc *sc;
606 
607 	sc = device_get_softc(dev);
608 
609 	PIC_UNMASK(root_pic, irq);
610 
611 	opal_call(OPAL_PCI_MSI_EOI, sc->phb_id, irq);
612 }
613 
614 
615