1 /*-
2 * Copyright (c) 2015-2016 The FreeBSD Foundation
3 * Copyright (c) 2023 Arm Ltd
4 *
5 * This software was developed by Andrew Turner under
6 * the sponsorship of the FreeBSD Foundation.
7 *
8 * This software was developed by Semihalf under
9 * the sponsorship of the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include "opt_acpi.h"
34 #include "opt_platform.h"
35 #include "opt_iommu.h"
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/bus.h>
40 #include <sys/cpuset.h>
41 #include <sys/domainset.h>
42 #include <sys/endian.h>
43 #include <sys/kernel.h>
44 #include <sys/lock.h>
45 #include <sys/malloc.h>
46 #include <sys/module.h>
47 #include <sys/mutex.h>
48 #include <sys/physmem.h>
49 #include <sys/proc.h>
50 #include <sys/taskqueue.h>
51 #include <sys/tree.h>
52 #include <sys/queue.h>
53 #include <sys/rman.h>
54 #include <sys/sbuf.h>
55 #include <sys/smp.h>
56 #include <sys/sysctl.h>
57 #include <sys/vmem.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 #include <vm/vm_page.h>
62
63 #include <machine/bus.h>
64 #include <machine/intr.h>
65
66 #include <arm/arm/gic_common.h>
67 #include <arm64/arm64/gic_v3_reg.h>
68 #include <arm64/arm64/gic_v3_var.h>
69
70 #ifdef FDT
71 #include <dev/ofw/openfirm.h>
72 #include <dev/ofw/ofw_bus.h>
73 #include <dev/ofw/ofw_bus_subr.h>
74 #endif
75 #include <dev/pci/pcireg.h>
76 #include <dev/pci/pcivar.h>
77
78 #ifdef IOMMU
79 #include <dev/iommu/iommu.h>
80 #include <dev/iommu/iommu_gas.h>
81 #endif
82
83 #include "pcib_if.h"
84 #include "pic_if.h"
85 #include "msi_if.h"
86
87 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
88 "ARM GICv3 Interrupt Translation Service");
89
90 #define LPI_NIRQS (64 * 1024)
91
92 /* The size and alignment of the command circular buffer */
93 #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */
94 #define ITS_CMDQ_ALIGN (64 * 1024)
95
96 #define LPI_CONFTAB_SIZE LPI_NIRQS
97 #define LPI_CONFTAB_ALIGN (64 * 1024)
98 #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */
99
100 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
101 #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
102 #define LPI_PENDTAB_ALIGN (64 * 1024)
103 #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */
104
105 #define LPI_INT_TRANS_TAB_ALIGN 256
106 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
107
108 /* ITS commands encoding */
109 #define ITS_CMD_MOVI (0x01)
110 #define ITS_CMD_SYNC (0x05)
111 #define ITS_CMD_MAPD (0x08)
112 #define ITS_CMD_MAPC (0x09)
113 #define ITS_CMD_MAPTI (0x0a)
114 #define ITS_CMD_MAPI (0x0b)
115 #define ITS_CMD_INV (0x0c)
116 #define ITS_CMD_INVALL (0x0d)
117 /* Command */
118 #define CMD_COMMAND_MASK (0xFFUL)
119 /* PCI device ID */
120 #define CMD_DEVID_SHIFT (32)
121 #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
122 /* Size of IRQ ID bitfield */
123 #define CMD_SIZE_MASK (0xFFUL)
124 /* Virtual LPI ID */
125 #define CMD_ID_MASK (0xFFFFFFFFUL)
126 /* Physical LPI ID */
127 #define CMD_PID_SHIFT (32)
128 #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT)
129 /* Collection */
130 #define CMD_COL_MASK (0xFFFFUL)
131 /* Target (CPU or Re-Distributor) */
132 #define CMD_TARGET_SHIFT (16)
133 #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
134 /* Interrupt Translation Table address */
135 #define CMD_ITT_MASK (0xFFFFFFFFFF00UL)
136 /* Valid command bit */
137 #define CMD_VALID_SHIFT (63)
138 #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT)
139
140 #define ITS_TARGET_NONE 0xFBADBEEF
141
142 /* LPI chunk owned by ITS device */
143 struct lpi_chunk {
144 u_int lpi_base;
145 u_int lpi_free; /* First free LPI in set */
146 u_int lpi_num; /* Total number of LPIs in chunk */
147 u_int lpi_busy; /* Number of busy LPIs in chink */
148 };
149
150 /* ITS device */
151 struct its_dev {
152 TAILQ_ENTRY(its_dev) entry;
153 /* PCI device */
154 device_t pci_dev;
155 /* Device ID (i.e. PCI device ID) */
156 uint32_t devid;
157 /* List of assigned LPIs */
158 struct lpi_chunk lpis;
159 /* Virtual address of ITT */
160 void *itt;
161 };
162
163 /*
164 * ITS command descriptor.
165 * Idea for command description passing taken from Linux.
166 */
167 struct its_cmd_desc {
168 uint8_t cmd_type;
169
170 union {
171 struct {
172 struct its_dev *its_dev;
173 struct its_col *col;
174 uint32_t id;
175 } cmd_desc_movi;
176
177 struct {
178 struct its_col *col;
179 } cmd_desc_sync;
180
181 struct {
182 struct its_col *col;
183 uint8_t valid;
184 } cmd_desc_mapc;
185
186 struct {
187 struct its_dev *its_dev;
188 struct its_col *col;
189 uint32_t pid;
190 uint32_t id;
191 } cmd_desc_mapvi;
192
193 struct {
194 struct its_dev *its_dev;
195 struct its_col *col;
196 uint32_t pid;
197 } cmd_desc_mapi;
198
199 struct {
200 struct its_dev *its_dev;
201 uint8_t valid;
202 } cmd_desc_mapd;
203
204 struct {
205 struct its_dev *its_dev;
206 struct its_col *col;
207 uint32_t pid;
208 } cmd_desc_inv;
209
210 struct {
211 struct its_col *col;
212 } cmd_desc_invall;
213 };
214 };
215
216 /* ITS command. Each command is 32 bytes long */
217 struct its_cmd {
218 uint64_t cmd_dword[4]; /* ITS command double word */
219 };
220
221 /* An ITS private table */
222 struct its_ptable {
223 void *ptab_vaddr;
224 /* Size of the L1 and L2 tables */
225 size_t ptab_l1_size;
226 size_t ptab_l2_size;
227 /* Number of L1 and L2 entries */
228 int ptab_l1_nidents;
229 int ptab_l2_nidents;
230
231 int ptab_page_size;
232 int ptab_share;
233 bool ptab_indirect;
234 };
235
236 /* ITS collection description. */
237 struct its_col {
238 uint64_t col_target; /* Target Re-Distributor */
239 uint64_t col_id; /* Collection ID */
240 };
241
242 struct gicv3_its_irqsrc {
243 struct intr_irqsrc gi_isrc;
244 u_int gi_id;
245 u_int gi_lpi;
246 struct its_dev *gi_its_dev;
247 TAILQ_ENTRY(gicv3_its_irqsrc) gi_link;
248 };
249
250 struct gicv3_its_softc {
251 device_t dev;
252 struct intr_pic *sc_pic;
253 struct resource *sc_its_res;
254
255 cpuset_t sc_cpus;
256 struct domainset *sc_ds;
257 u_int gic_irq_cpu;
258 int sc_devbits;
259 int sc_dev_table_idx;
260
261 struct its_ptable sc_its_ptab[GITS_BASER_NUM];
262 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */
263
264 /*
265 * TODO: We should get these from the parent as we only want a
266 * single copy of each across the interrupt controller.
267 */
268 uint8_t *sc_conf_base;
269 void *sc_pend_base[MAXCPU];
270
271 /* Command handling */
272 struct mtx sc_its_cmd_lock;
273 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
274 size_t sc_its_cmd_next_idx;
275
276 vmem_t *sc_irq_alloc;
277 struct gicv3_its_irqsrc **sc_irqs;
278 u_int sc_irq_base;
279 u_int sc_irq_length;
280 u_int sc_irq_count;
281
282 struct mtx sc_its_dev_lock;
283 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
284 TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs;
285
286 #define ITS_FLAGS_CMDQ_FLUSH 0x00000001
287 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002
288 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004
289 #define ITS_FLAGS_LPI_PREALLOC 0x00000008
290 #define ITS_FLAGS_FORCE_NOSHAREABLE 0x00000010
291 u_int sc_its_flags;
292 bool trace_enable;
293 vm_page_t ma; /* fake msi page */
294 vm_paddr_t malloc_max_addr; /* max address for contigmalloc */
295 };
296
297 typedef bool (its_quirk_detect_t)(device_t);
298 typedef void (its_quirk_func_t)(device_t);
299
300 static its_quirk_detect_t its_detect_cavium_22375;
301 static its_quirk_func_t its_quirk_cavium_22375;
302 #ifdef FDT
303 static its_quirk_detect_t its_detect_rk356x;
304 static its_quirk_func_t its_quirk_rk356x;
305 static its_quirk_detect_t its_detect_rk3588;
306 static its_quirk_func_t its_quirk_rk3588;
307 #endif
308
309 static const struct {
310 const char *desc;
311 its_quirk_detect_t *detect;
312 its_quirk_func_t *func;
313 } its_quirks[] = {
314 {
315 /* Cavium ThunderX Pass 1.x */
316 .desc = "Cavium ThunderX errata: 22375, 24313",
317 .detect = its_detect_cavium_22375,
318 .func = its_quirk_cavium_22375,
319 },
320 #ifdef FDT
321 {
322 /* Rockchip RK356X implementation bugs */
323 .desc = "RK356X ITS errata",
324 .detect = its_detect_rk356x,
325 .func = its_quirk_rk356x,
326 },
327 {
328 /* Rockchip RK3588 implementation bugs */
329 .desc = "RK3588 ITS errata",
330 .detect = its_detect_rk3588,
331 .func = its_quirk_rk3588,
332 },
333 #endif
334 };
335
336 #define gic_its_read_4(sc, reg) \
337 bus_read_4((sc)->sc_its_res, (reg))
338 #define gic_its_read_8(sc, reg) \
339 bus_read_8((sc)->sc_its_res, (reg))
340
341 #define gic_its_write_4(sc, reg, val) \
342 bus_write_4((sc)->sc_its_res, (reg), (val))
343 #define gic_its_write_8(sc, reg, val) \
344 bus_write_8((sc)->sc_its_res, (reg), (val))
345
346 static device_attach_t gicv3_its_attach;
347 static device_detach_t gicv3_its_detach;
348
349 static pic_disable_intr_t gicv3_its_disable_intr;
350 static pic_enable_intr_t gicv3_its_enable_intr;
351 static pic_map_intr_t gicv3_its_map_intr;
352 static pic_setup_intr_t gicv3_its_setup_intr;
353 static pic_post_filter_t gicv3_its_post_filter;
354 static pic_post_ithread_t gicv3_its_post_ithread;
355 static pic_pre_ithread_t gicv3_its_pre_ithread;
356 static pic_bind_intr_t gicv3_its_bind_intr;
357 #ifdef SMP
358 static pic_init_secondary_t gicv3_its_init_secondary;
359 #endif
360 static msi_alloc_msi_t gicv3_its_alloc_msi;
361 static msi_release_msi_t gicv3_its_release_msi;
362 static msi_alloc_msix_t gicv3_its_alloc_msix;
363 static msi_release_msix_t gicv3_its_release_msix;
364 static msi_map_msi_t gicv3_its_map_msi;
365 #ifdef IOMMU
366 static msi_iommu_init_t gicv3_iommu_init;
367 static msi_iommu_deinit_t gicv3_iommu_deinit;
368 #endif
369
370 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
371 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
372 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
373 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
374 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
375 static void its_cmd_invall(device_t, struct its_col *);
376
377 static device_method_t gicv3_its_methods[] = {
378 /* Device interface */
379 DEVMETHOD(device_detach, gicv3_its_detach),
380
381 /* Interrupt controller interface */
382 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr),
383 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr),
384 DEVMETHOD(pic_map_intr, gicv3_its_map_intr),
385 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr),
386 DEVMETHOD(pic_post_filter, gicv3_its_post_filter),
387 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread),
388 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread),
389 #ifdef SMP
390 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr),
391 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary),
392 #endif
393
394 /* MSI/MSI-X */
395 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi),
396 DEVMETHOD(msi_release_msi, gicv3_its_release_msi),
397 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix),
398 DEVMETHOD(msi_release_msix, gicv3_its_release_msix),
399 DEVMETHOD(msi_map_msi, gicv3_its_map_msi),
400 #ifdef IOMMU
401 DEVMETHOD(msi_iommu_init, gicv3_iommu_init),
402 DEVMETHOD(msi_iommu_deinit, gicv3_iommu_deinit),
403 #endif
404
405 /* End */
406 DEVMETHOD_END
407 };
408
409 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
410 sizeof(struct gicv3_its_softc));
411
412 /* Limit maximum address for memory mapped tables and buffers */
413 static vm_paddr_t
gicv3_its_limit_max_addr(struct gicv3_its_softc * sc,vm_paddr_t addr)414 gicv3_its_limit_max_addr(struct gicv3_its_softc *sc, vm_paddr_t addr)
415 {
416 return (sc->malloc_max_addr > addr ? addr: sc->malloc_max_addr);
417 }
418
419 static void
gicv3_its_cmdq_init(struct gicv3_its_softc * sc)420 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
421 {
422 vm_paddr_t cmd_paddr;
423 uint64_t reg, tmp;
424
425 /* Set up the command circular buffer */
426 sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS,
427 sc->sc_ds, M_WAITOK | M_ZERO, 0,
428 gicv3_its_limit_max_addr(sc, (1ul << 48) - 1), ITS_CMDQ_ALIGN,
429 0);
430 sc->sc_its_cmd_next_idx = 0;
431
432 cmd_paddr = vtophys(sc->sc_its_cmd_base);
433
434 /* Set the base of the command buffer */
435 reg = GITS_CBASER_VALID |
436 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
437 cmd_paddr | (ITS_CMDQ_SIZE / 4096 - 1);
438 if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE)
439 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
440 else
441 reg |= GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT;
442 gic_its_write_8(sc, GITS_CBASER, reg);
443
444 /* Read back to check for fixed value fields */
445 tmp = gic_its_read_8(sc, GITS_CBASER);
446
447 if ((tmp & GITS_CBASER_SHARE_MASK) !=
448 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
449 /* Check if the hardware reported non-shareable */
450 if ((tmp & GITS_CBASER_SHARE_MASK) ==
451 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
452 /* If so remove the cache attribute */
453 reg &= ~GITS_CBASER_CACHE_MASK;
454 reg &= ~GITS_CBASER_SHARE_MASK;
455 /* Set to Non-cacheable, Non-shareable */
456 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
457 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
458
459 gic_its_write_8(sc, GITS_CBASER, reg);
460 }
461
462 /* The command queue has to be flushed after each command */
463 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
464 }
465
466 /* Get the next command from the start of the buffer */
467 gic_its_write_8(sc, GITS_CWRITER, 0x0);
468 }
469
470 static int
gicv3_its_table_page_size(struct gicv3_its_softc * sc,int table)471 gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table)
472 {
473 uint64_t reg, tmp;
474 int page_size;
475
476 page_size = PAGE_SIZE_64K;
477 reg = gic_its_read_8(sc, GITS_BASER(table));
478
479 while (1) {
480 reg &= ~GITS_BASER_PSZ_MASK;
481 switch (page_size) {
482 case PAGE_SIZE_4K: /* 4KB */
483 reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
484 break;
485 case PAGE_SIZE_16K: /* 16KB */
486 reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
487 break;
488 case PAGE_SIZE_64K: /* 64KB */
489 reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
490 break;
491 }
492
493 /* Write the new page size */
494 gic_its_write_8(sc, GITS_BASER(table), reg);
495
496 /* Read back to check */
497 tmp = gic_its_read_8(sc, GITS_BASER(table));
498
499 /* The page size is correct */
500 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK))
501 return (page_size);
502
503 switch (page_size) {
504 default:
505 return (-1);
506 case PAGE_SIZE_16K:
507 page_size = PAGE_SIZE_4K;
508 break;
509 case PAGE_SIZE_64K:
510 page_size = PAGE_SIZE_16K;
511 break;
512 }
513 }
514 }
515
516 static bool
gicv3_its_table_supports_indirect(struct gicv3_its_softc * sc,int table)517 gicv3_its_table_supports_indirect(struct gicv3_its_softc *sc, int table)
518 {
519 uint64_t reg;
520
521 reg = gic_its_read_8(sc, GITS_BASER(table));
522
523 /* Try setting the indirect flag */
524 reg |= GITS_BASER_INDIRECT;
525 gic_its_write_8(sc, GITS_BASER(table), reg);
526
527 /* Read back to check */
528 reg = gic_its_read_8(sc, GITS_BASER(table));
529 return ((reg & GITS_BASER_INDIRECT) != 0);
530 }
531
532
533 static int
gicv3_its_table_init(device_t dev,struct gicv3_its_softc * sc)534 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
535 {
536 void *table;
537 vm_paddr_t paddr;
538 uint64_t cache, reg, share, tmp, type;
539 size_t its_tbl_size, nitspages, npages;
540 size_t l1_esize, l2_esize, l1_nidents, l2_nidents;
541 int i, page_size;
542 int devbits;
543 bool indirect;
544
545 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
546 /*
547 * GITS_TYPER[17:13] of ThunderX reports that device IDs
548 * are to be 21 bits in length. The entry size of the ITS
549 * table can be read from GITS_BASERn[52:48] and on ThunderX
550 * is supposed to be 8 bytes in length (for device table).
551 * Finally the page size that is to be used by ITS to access
552 * this table will be set to 64KB.
553 *
554 * This gives 0x200000 entries of size 0x8 bytes covered by
555 * 256 pages each of which 64KB in size. The number of pages
556 * (minus 1) should then be written to GITS_BASERn[7:0]. In
557 * that case this value would be 0xFF but on ThunderX the
558 * maximum value that HW accepts is 0xFD.
559 *
560 * Set an arbitrary number of device ID bits to 20 in order
561 * to limit the number of entries in ITS device table to
562 * 0x100000 and the table size to 8MB.
563 */
564 devbits = 20;
565 cache = 0;
566 } else {
567 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
568 if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE)
569 cache = GITS_BASER_CACHE_NC;
570 else
571 cache = GITS_BASER_CACHE_RAWAWB;
572 }
573 sc->sc_devbits = devbits;
574
575 if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE)
576 share = GITS_BASER_SHARE_NS;
577 else
578 share = GITS_BASER_SHARE_IS;
579
580 for (i = 0; i < GITS_BASER_NUM; i++) {
581 reg = gic_its_read_8(sc, GITS_BASER(i));
582 /* The type of table */
583 type = GITS_BASER_TYPE(reg);
584 if (type == GITS_BASER_TYPE_UNIMPL)
585 continue;
586
587 /* The table entry size */
588 l1_esize = GITS_BASER_ESIZE(reg);
589
590 /* Find the tables page size */
591 page_size = gicv3_its_table_page_size(sc, i);
592 if (page_size == -1) {
593 device_printf(dev, "No valid page size for table %d\n",
594 i);
595 return (EINVAL);
596 }
597
598 indirect = false;
599 l2_nidents = 0;
600 l2_esize = 0;
601 switch(type) {
602 case GITS_BASER_TYPE_DEV:
603 if (sc->sc_dev_table_idx != -1)
604 device_printf(dev,
605 "Warning: Multiple device tables found\n");
606
607 sc->sc_dev_table_idx = i;
608 l1_nidents = (1 << devbits);
609 if ((l1_esize * l1_nidents) > (page_size * 2)) {
610 indirect =
611 gicv3_its_table_supports_indirect(sc, i);
612 if (indirect) {
613 /*
614 * Each l1 entry is 8 bytes and points
615 * to an l2 table of size page_size.
616 * Calculate how many entries this is
617 * and use this to find how many
618 * 8 byte l1 idents we need.
619 */
620 l2_esize = l1_esize;
621 l2_nidents = page_size / l2_esize;
622 l1_nidents = l1_nidents / l2_nidents;
623 l1_esize = GITS_INDIRECT_L1_ESIZE;
624 }
625 }
626 its_tbl_size = l1_esize * l1_nidents;
627 its_tbl_size = roundup2(its_tbl_size, page_size);
628 break;
629 case GITS_BASER_TYPE_PP: /* Undocumented? */
630 case GITS_BASER_TYPE_IC:
631 its_tbl_size = page_size;
632 break;
633 case GITS_BASER_TYPE_VP:
634 /*
635 * If GITS_TYPER.SVPET != 0, the pending table is
636 * shared amongst the redistibutors and ther other
637 * ITSes. Requiring sharing across the ITSes when none
638 * of the redistributors have GICR_VPROPBASER.Valid==1
639 * isn't specified in the architecture, but that's how
640 * the GIC-700 behaves. We don't handle vPE tables at
641 * all yet, so just skip this base register.
642 */
643 default:
644 if (bootverbose)
645 device_printf(dev, "Unhandled table type %lx\n",
646 type);
647 continue;
648 }
649 npages = howmany(its_tbl_size, PAGE_SIZE);
650
651 /* Allocate the table */
652 table = contigmalloc_domainset(npages * PAGE_SIZE,
653 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0,
654 gicv3_its_limit_max_addr(sc, (1ul << 48) - 1),
655 PAGE_SIZE_64K, 0);
656
657 sc->sc_its_ptab[i].ptab_vaddr = table;
658 sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size;
659 sc->sc_its_ptab[i].ptab_l1_nidents = l1_nidents;
660 sc->sc_its_ptab[i].ptab_l2_size = page_size;
661 sc->sc_its_ptab[i].ptab_l2_nidents = l2_nidents;
662
663 sc->sc_its_ptab[i].ptab_indirect = indirect;
664 sc->sc_its_ptab[i].ptab_page_size = page_size;
665
666 paddr = vtophys(table);
667
668 while (1) {
669 nitspages = howmany(its_tbl_size, page_size);
670
671 /* Clear the fields we will be setting */
672 reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT |
673 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
674 GITS_BASER_PA_MASK |
675 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
676 GITS_BASER_SIZE_MASK);
677 /* Set the new values */
678 reg |= GITS_BASER_VALID |
679 (indirect ? GITS_BASER_INDIRECT : 0) |
680 (cache << GITS_BASER_CACHE_SHIFT) |
681 (type << GITS_BASER_TYPE_SHIFT) |
682 paddr | (share << GITS_BASER_SHARE_SHIFT) |
683 (nitspages - 1);
684
685 switch (page_size) {
686 case PAGE_SIZE_4K: /* 4KB */
687 reg |=
688 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
689 break;
690 case PAGE_SIZE_16K: /* 16KB */
691 reg |=
692 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
693 break;
694 case PAGE_SIZE_64K: /* 64KB */
695 reg |=
696 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
697 break;
698 }
699
700 gic_its_write_8(sc, GITS_BASER(i), reg);
701
702 /* Read back to check */
703 tmp = gic_its_read_8(sc, GITS_BASER(i));
704
705 /* Do the shareability masks line up? */
706 if ((tmp & GITS_BASER_SHARE_MASK) !=
707 (reg & GITS_BASER_SHARE_MASK)) {
708 share = (tmp & GITS_BASER_SHARE_MASK) >>
709 GITS_BASER_SHARE_SHIFT;
710 continue;
711 }
712
713 if (tmp != reg) {
714 device_printf(dev, "GITS_BASER%d: "
715 "unable to be updated: %lx != %lx\n",
716 i, reg, tmp);
717 return (ENXIO);
718 }
719
720 sc->sc_its_ptab[i].ptab_share = share;
721 /* We should have made all needed changes */
722 break;
723 }
724 }
725
726 return (0);
727 }
728
729 static void
gicv3_its_conftable_init(struct gicv3_its_softc * sc)730 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
731 {
732 /* note: we assume the ITS children are serialized by the parent */
733 static void *conf_table;
734 int extra_flags = 0;
735 device_t gicv3;
736 uint32_t ctlr;
737 vm_paddr_t conf_pa;
738 vm_offset_t conf_va;
739
740 /*
741 * The PROPBASER is a singleton in our parent. We only set it up the
742 * first time through. conf_table is effectively global to all the units
743 * and we rely on subr_bus to serialize probe/attach.
744 */
745 if (conf_table != NULL) {
746 sc->sc_conf_base = conf_table;
747 return;
748 }
749
750 gicv3 = device_get_parent(sc->dev);
751 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
752 if ((ctlr & GICR_CTLR_LPI_ENABLE) != 0) {
753 conf_pa = gic_r_read_8(gicv3, GICR_PROPBASER);
754 conf_pa &= GICR_PROPBASER_PA_MASK;
755 /*
756 * If there was a pre-existing PROPBASER, then we need to honor
757 * it because implementation defined behavior in gicv3 makes it
758 * impossible to quiesce to change it out. We will only see a
759 * pre-existing one when we've been kexec'd from a Linux kernel,
760 * or from a LinuxBoot environment.
761 *
762 * Linux provides us with a MEMRESERVE table that we put into
763 * the excluded physmem area. If PROPBASER isn't in this tabke,
764 * the system cannot run due to random memory corruption,
765 * so we panic for this case.
766 */
767 if (!physmem_excluded(conf_pa, LPI_CONFTAB_SIZE))
768 panic("gicv3 PROPBASER needs to reuse %#lx, but not reserved",
769 conf_pa);
770 conf_va = PHYS_TO_DMAP(conf_pa);
771 if (!pmap_klookup(conf_va, NULL))
772 panic("Cannot map prior LPI mapping into KVA");
773 conf_table = (void *)conf_va;
774 extra_flags = ITS_FLAGS_LPI_PREALLOC | ITS_FLAGS_LPI_CONF_FLUSH;
775 if (bootverbose)
776 device_printf(sc->dev,
777 "LPI enabled, conf table using pa %#lx va %lx\n",
778 conf_pa, conf_va);
779 } else {
780 /*
781 * Otherwise just allocate contiguous pages. We'll configure the
782 * PROPBASER register later in its_init_cpu_lpi().
783 */
784 conf_table = contigmalloc(LPI_CONFTAB_SIZE,
785 M_GICV3_ITS, M_WAITOK, 0,
786 gicv3_its_limit_max_addr(sc, LPI_CONFTAB_MAX_ADDR),
787 LPI_CONFTAB_ALIGN, 0);
788 }
789 sc->sc_conf_base = conf_table;
790 sc->sc_its_flags |= extra_flags;
791
792 /* Set the default configuration */
793 memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
794 LPI_CONFTAB_SIZE);
795
796 /* Flush the table to memory */
797 cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE);
798 }
799
800 static void
gicv3_its_pendtables_init(struct gicv3_its_softc * sc)801 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
802 {
803
804 if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) {
805 for (int i = 0; i <= mp_maxid; i++) {
806 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
807 continue;
808
809 sc->sc_pend_base[i] = contigmalloc(
810 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
811 0,
812 gicv3_its_limit_max_addr(sc, LPI_PENDTAB_MAX_ADDR),
813 LPI_PENDTAB_ALIGN, 0);
814
815 /* Flush so the ITS can see the memory */
816 cpu_dcache_wb_range(sc->sc_pend_base[i],
817 LPI_PENDTAB_SIZE);
818 }
819 }
820 }
821
822 static void
its_init_cpu_lpi(device_t dev,struct gicv3_its_softc * sc)823 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc)
824 {
825 device_t gicv3;
826 uint64_t xbaser, tmp, size;
827 uint32_t ctlr;
828 u_int cpuid;
829
830 gicv3 = device_get_parent(dev);
831 cpuid = PCPU_GET(cpuid);
832
833 /*
834 * Set the redistributor base. If we're reusing what we found on boot
835 * since the gic was already running, then don't touch it here. We also
836 * don't need to disable / enable LPI if we're not changing PROPBASER,
837 * so only do that if we're not prealloced.
838 */
839 if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) {
840 /* Disable LPIs */
841 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
842 ctlr &= ~GICR_CTLR_LPI_ENABLE;
843 gic_r_write_4(gicv3, GICR_CTLR, ctlr);
844
845 /* Make sure changes are observable my the GIC */
846 dsb(sy);
847
848 size = ilog2_long(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1;
849
850 xbaser = vtophys(sc->sc_conf_base) |
851 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
852 size;
853 if (gicv3_get_flags(sc->dev) & GIC_V3_FLAGS_FORCE_NOSHAREABLE)
854 xbaser |= GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT;
855 else
856 xbaser |= GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT;
857 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
858
859 /* Check the cache attributes we set */
860 tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
861
862 if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
863 (xbaser & GICR_PROPBASER_SHARE_MASK)) {
864 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
865 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
866 /* We need to mark as non-cacheable */
867 xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
868 GICR_PROPBASER_CACHE_MASK);
869 /* Non-cacheable */
870 xbaser |= GICR_PROPBASER_CACHE_NIN <<
871 GICR_PROPBASER_CACHE_SHIFT;
872 /* Non-shareable */
873 xbaser |= GICR_PROPBASER_SHARE_NS <<
874 GICR_PROPBASER_SHARE_SHIFT;
875 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
876 }
877 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
878 }
879
880 /*
881 * Set the LPI pending table base
882 */
883 xbaser = vtophys(sc->sc_pend_base[cpuid]) |
884 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT);
885 if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE)
886 xbaser |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
887 else
888 xbaser |= GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT;
889
890 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
891
892 tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
893
894 if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
895 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
896 /* Clear the cahce and shareability bits */
897 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
898 GICR_PENDBASER_SHARE_MASK);
899 /* Mark as non-shareable */
900 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
901 /* And non-cacheable */
902 xbaser |= GICR_PENDBASER_CACHE_NIN <<
903 GICR_PENDBASER_CACHE_SHIFT;
904 }
905
906 /* Enable LPIs */
907 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
908 ctlr |= GICR_CTLR_LPI_ENABLE;
909 gic_r_write_4(gicv3, GICR_CTLR, ctlr);
910
911 /* Make sure the GIC has seen everything */
912 dsb(sy);
913 } else {
914 KASSERT(sc->sc_pend_base[cpuid] == NULL,
915 ("PREALLOC too soon cpuid %d", cpuid));
916 tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
917 tmp &= GICR_PENDBASER_PA_MASK;
918 if (!physmem_excluded(tmp, LPI_PENDTAB_SIZE))
919 panic("gicv3 PENDBASER on cpu %d needs to reuse 0x%#lx, but not reserved\n",
920 cpuid, tmp);
921 sc->sc_pend_base[cpuid] = (void *)PHYS_TO_DMAP(tmp);
922 }
923
924
925 if (bootverbose)
926 device_printf(gicv3, "using %sPENDBASE of %#lx on cpu %d\n",
927 (sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) ? "pre-existing " : "",
928 vtophys(sc->sc_pend_base[cpuid]), cpuid);
929 }
930
931 static int
its_init_cpu(device_t dev,struct gicv3_its_softc * sc)932 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
933 {
934 device_t gicv3;
935 vm_paddr_t target;
936 u_int cpuid;
937 struct redist_pcpu *rpcpu;
938
939 gicv3 = device_get_parent(dev);
940 cpuid = PCPU_GET(cpuid);
941 if (!CPU_ISSET(cpuid, &sc->sc_cpus))
942 return (0);
943
944 /* Check if the ITS is enabled on this CPU */
945 if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0)
946 return (ENXIO);
947
948 rpcpu = gicv3_get_redist(dev);
949
950 /* Do per-cpu LPI init once */
951 if (!rpcpu->lpi_enabled) {
952 its_init_cpu_lpi(dev, sc);
953 rpcpu->lpi_enabled = true;
954 }
955
956 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
957 /* This ITS wants the redistributor physical address */
958 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) +
959 rpcpu->offset);
960 } else {
961 /* This ITS wants the unique processor number */
962 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) <<
963 CMD_TARGET_SHIFT;
964 }
965
966 sc->sc_its_cols[cpuid]->col_target = target;
967 sc->sc_its_cols[cpuid]->col_id = cpuid;
968
969 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
970 its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
971
972 return (0);
973 }
974
975 static int
gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)976 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)
977 {
978 struct gicv3_its_softc *sc;
979 int rv;
980
981 sc = arg1;
982
983 rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req);
984 if (rv != 0 || req->newptr == NULL)
985 return (rv);
986 if (sc->trace_enable)
987 gic_its_write_8(sc, GITS_TRKCTLR, 3);
988 else
989 gic_its_write_8(sc, GITS_TRKCTLR, 0);
990
991 return (0);
992 }
993
994 static int
gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)995 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)
996 {
997 struct gicv3_its_softc *sc;
998 struct sbuf *sb;
999 int err;
1000
1001 sc = arg1;
1002 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
1003 if (sb == NULL) {
1004 device_printf(sc->dev, "Could not allocate sbuf for output.\n");
1005 return (ENOMEM);
1006 }
1007 sbuf_cat(sb, "\n");
1008 sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n",
1009 gic_its_read_4(sc, GITS_TRKCTLR));
1010 sbuf_printf(sb, "GITS_TRKR: 0x%08X\n",
1011 gic_its_read_4(sc, GITS_TRKR));
1012 sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n",
1013 gic_its_read_4(sc, GITS_TRKDIDR));
1014 sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n",
1015 gic_its_read_4(sc, GITS_TRKPIDR));
1016 sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n",
1017 gic_its_read_4(sc, GITS_TRKVIDR));
1018 sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n",
1019 gic_its_read_4(sc, GITS_TRKTGTR));
1020
1021 err = sbuf_finish(sb);
1022 if (err)
1023 device_printf(sc->dev, "Error finishing sbuf: %d\n", err);
1024 sbuf_delete(sb);
1025 return(err);
1026 }
1027
1028 static int
gicv3_its_init_sysctl(struct gicv3_its_softc * sc)1029 gicv3_its_init_sysctl(struct gicv3_its_softc *sc)
1030 {
1031 struct sysctl_oid *oid, *child;
1032 struct sysctl_ctx_list *ctx_list;
1033
1034 ctx_list = device_get_sysctl_ctx(sc->dev);
1035 child = device_get_sysctl_tree(sc->dev);
1036 oid = SYSCTL_ADD_NODE(ctx_list,
1037 SYSCTL_CHILDREN(child), OID_AUTO, "tracing",
1038 CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing");
1039 if (oid == NULL)
1040 return (ENXIO);
1041
1042 /* Add registers */
1043 SYSCTL_ADD_PROC(ctx_list,
1044 SYSCTL_CHILDREN(oid), OID_AUTO, "enable",
1045 CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
1046 gicv3_its_sysctl_trace_enable, "CU", "Enable tracing");
1047 SYSCTL_ADD_PROC(ctx_list,
1048 SYSCTL_CHILDREN(oid), OID_AUTO, "capture",
1049 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
1050 gicv3_its_sysctl_trace_regs, "", "Captured tracing registers.");
1051
1052 return (0);
1053 }
1054
1055 static int
gicv3_its_attach(device_t dev)1056 gicv3_its_attach(device_t dev)
1057 {
1058 struct gicv3_its_softc *sc;
1059 int domain, err, i, rid;
1060 uint64_t phys;
1061 uint32_t ctlr, iidr;
1062
1063 sc = device_get_softc(dev);
1064
1065 sc->sc_dev_table_idx = -1;
1066 sc->sc_irq_length = gicv3_get_nirqs(dev);
1067 sc->sc_irq_base = GIC_FIRST_LPI;
1068 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
1069 sc->malloc_max_addr = ~0;
1070
1071 rid = 0;
1072 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1073 RF_ACTIVE);
1074 if (sc->sc_its_res == NULL) {
1075 device_printf(dev, "Could not allocate memory\n");
1076 return (ENXIO);
1077 }
1078
1079 phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
1080 GITS_TRANSLATER, PAGE_SIZE);
1081 sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO);
1082 vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);
1083
1084 CPU_COPY(&all_cpus, &sc->sc_cpus);
1085 iidr = gic_its_read_4(sc, GITS_IIDR);
1086 for (i = 0; i < nitems(its_quirks); i++) {
1087 if (its_quirks[i].detect(dev)) {
1088 if (bootverbose) {
1089 device_printf(dev, "Applying %s\n",
1090 its_quirks[i].desc);
1091 }
1092 its_quirks[i].func(dev);
1093 break;
1094 }
1095 }
1096
1097 if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) {
1098 sc->sc_ds = DOMAINSET_PREF(domain);
1099 } else {
1100 sc->sc_ds = DOMAINSET_RR();
1101 }
1102
1103 /*
1104 * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be
1105 * coming in via, for instance, a kexec/kboot style setup where a
1106 * previous kernel has configured then relinquished control. Clear it
1107 * so that we can reconfigure GITS_BASER*.
1108 */
1109 ctlr = gic_its_read_4(sc, GITS_CTLR);
1110 if ((ctlr & GITS_CTLR_EN) != 0) {
1111 ctlr &= ~GITS_CTLR_EN;
1112 gic_its_write_4(sc, GITS_CTLR, ctlr);
1113 }
1114
1115 /* Allocate the private tables */
1116 err = gicv3_its_table_init(dev, sc);
1117 if (err != 0)
1118 return (err);
1119
1120 /* Protects access to the device list */
1121 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
1122
1123 /* Protects access to the ITS command circular buffer. */
1124 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
1125
1126 /* Allocate the command circular buffer */
1127 gicv3_its_cmdq_init(sc);
1128
1129 /* Allocate the per-CPU collections */
1130 for (int cpu = 0; cpu <= mp_maxid; cpu++)
1131 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
1132 sc->sc_its_cols[cpu] = malloc_domainset(
1133 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
1134 DOMAINSET_PREF(pcpu_find(cpu)->pc_domain),
1135 M_WAITOK | M_ZERO);
1136 else
1137 sc->sc_its_cols[cpu] = NULL;
1138
1139 /* Enable the ITS */
1140 gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN);
1141
1142 /* Create the LPI configuration table */
1143 gicv3_its_conftable_init(sc);
1144
1145 /* And the pending tebles */
1146 gicv3_its_pendtables_init(sc);
1147
1148 /* Enable LPIs on this CPU */
1149 its_init_cpu(dev, sc);
1150
1151 TAILQ_INIT(&sc->sc_its_dev_list);
1152 TAILQ_INIT(&sc->sc_free_irqs);
1153
1154 /*
1155 * Create the vmem object to allocate INTRNG IRQs from. We try to
1156 * use all IRQs not already used by the GICv3.
1157 * XXX: This assumes there are no other interrupt controllers in the
1158 * system.
1159 */
1160 sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0,
1161 gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK);
1162
1163 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
1164 M_GICV3_ITS, M_WAITOK | M_ZERO);
1165
1166 /* For GIC-500 install tracking sysctls. */
1167 if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) ==
1168 GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0))
1169 gicv3_its_init_sysctl(sc);
1170
1171 return (0);
1172 }
1173
1174 static int
gicv3_its_detach(device_t dev)1175 gicv3_its_detach(device_t dev)
1176 {
1177
1178 return (ENXIO);
1179 }
1180
1181 static bool
its_detect_cavium_22375(device_t dev)1182 its_detect_cavium_22375(device_t dev)
1183 {
1184 uint32_t iidr;
1185 struct gicv3_its_softc *sc;
1186
1187 sc = device_get_softc(dev);
1188 iidr = gic_its_read_4(sc, GITS_IIDR);
1189 if ((iidr & ~GITS_IIDR_REVISION_MASK) ==
1190 GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, GITS_IIDR_PROD_THUNDER,
1191 GITS_IIDR_VAR_THUNDER_1, 0))
1192 return (true);
1193 return(false);
1194 }
1195
1196 static void
its_quirk_cavium_22375(device_t dev)1197 its_quirk_cavium_22375(device_t dev)
1198 {
1199 struct gicv3_its_softc *sc;
1200 int domain;
1201
1202 sc = device_get_softc(dev);
1203 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
1204
1205 /*
1206 * We need to limit which CPUs we send these interrupts to on
1207 * the original dual socket ThunderX as it is unable to
1208 * forward them between the two sockets.
1209 */
1210 if (bus_get_domain(dev, &domain) == 0) {
1211 if (domain < MAXMEMDOM) {
1212 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
1213 } else {
1214 CPU_ZERO(&sc->sc_cpus);
1215 }
1216 }
1217 }
1218
1219 #ifdef FDT
1220 static bool
its_detect_rk356x(device_t dev)1221 its_detect_rk356x(device_t dev)
1222 {
1223
1224 if (ofw_bus_is_machine_compatible("rockchip,rk3566") ||
1225 ofw_bus_is_machine_compatible("rockchip,rk3568"))
1226 return (true);
1227 return(false);
1228 }
1229
1230 static void
its_quirk_rk356x(device_t dev)1231 its_quirk_rk356x(device_t dev)
1232 {
1233 struct gicv3_its_softc *sc;
1234
1235 sc = device_get_softc(dev);
1236 sc->malloc_max_addr = (1ul << 32) - 1;
1237 }
1238
1239 static bool
its_detect_rk3588(device_t dev)1240 its_detect_rk3588(device_t dev)
1241 {
1242
1243 if (ofw_bus_is_machine_compatible("rockchip,rk3588") ||
1244 ofw_bus_is_machine_compatible("rockchip,rk3588s"))
1245 return (true);
1246 return(false);
1247 }
1248
1249 static void
its_quirk_rk3588(device_t dev)1250 its_quirk_rk3588(device_t dev)
1251 {
1252 struct gicv3_its_softc *sc;
1253
1254 sc = device_get_softc(dev);
1255 sc->sc_its_flags |= ITS_FLAGS_FORCE_NOSHAREABLE;
1256 }
1257 #endif
1258
1259 static void
gicv3_its_disable_intr(device_t dev,struct intr_irqsrc * isrc)1260 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
1261 {
1262 struct gicv3_its_softc *sc;
1263 struct gicv3_its_irqsrc *girq;
1264 uint8_t *conf;
1265
1266 sc = device_get_softc(dev);
1267 girq = (struct gicv3_its_irqsrc *)isrc;
1268 conf = sc->sc_conf_base;
1269
1270 conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE;
1271
1272 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1273 /* Clean D-cache under command. */
1274 cpu_dcache_wb_range(&conf[girq->gi_lpi], 1);
1275 } else {
1276 /* DSB inner shareable, store */
1277 dsb(ishst);
1278 }
1279
1280 its_cmd_inv(dev, girq->gi_its_dev, girq);
1281 }
1282
1283 static void
gicv3_its_enable_intr(device_t dev,struct intr_irqsrc * isrc)1284 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
1285 {
1286 struct gicv3_its_softc *sc;
1287 struct gicv3_its_irqsrc *girq;
1288 uint8_t *conf;
1289
1290 sc = device_get_softc(dev);
1291 girq = (struct gicv3_its_irqsrc *)isrc;
1292 conf = sc->sc_conf_base;
1293
1294 conf[girq->gi_lpi] |= LPI_CONF_ENABLE;
1295
1296 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1297 /* Clean D-cache under command. */
1298 cpu_dcache_wb_range(&conf[girq->gi_lpi], 1);
1299 } else {
1300 /* DSB inner shareable, store */
1301 dsb(ishst);
1302 }
1303
1304 its_cmd_inv(dev, girq->gi_its_dev, girq);
1305 }
1306
1307 static int
gicv3_its_intr(void * arg,uintptr_t irq)1308 gicv3_its_intr(void *arg, uintptr_t irq)
1309 {
1310 struct gicv3_its_softc *sc = arg;
1311 struct gicv3_its_irqsrc *girq;
1312 struct trapframe *tf;
1313
1314 irq -= sc->sc_irq_base;
1315 girq = sc->sc_irqs[irq];
1316 if (girq == NULL)
1317 panic("gicv3_its_intr: Invalid interrupt %ld",
1318 irq + sc->sc_irq_base);
1319
1320 tf = curthread->td_intr_frame;
1321 intr_isrc_dispatch(&girq->gi_isrc, tf);
1322 return (FILTER_HANDLED);
1323 }
1324
1325 static void
gicv3_its_pre_ithread(device_t dev,struct intr_irqsrc * isrc)1326 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1327 {
1328 struct gicv3_its_irqsrc *girq;
1329
1330 girq = (struct gicv3_its_irqsrc *)isrc;
1331 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1332 }
1333
1334 static void
gicv3_its_post_ithread(device_t dev,struct intr_irqsrc * isrc)1335 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1336 {
1337
1338 }
1339
1340 static void
gicv3_its_post_filter(device_t dev,struct intr_irqsrc * isrc)1341 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
1342 {
1343 struct gicv3_its_irqsrc *girq;
1344
1345 girq = (struct gicv3_its_irqsrc *)isrc;
1346 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1347 }
1348
1349 static int
gicv3_its_select_cpu(device_t dev,struct intr_irqsrc * isrc)1350 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc)
1351 {
1352 struct gicv3_its_softc *sc;
1353
1354 sc = device_get_softc(dev);
1355 if (CPU_EMPTY(&isrc->isrc_cpu)) {
1356 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
1357 &sc->sc_cpus);
1358 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
1359 }
1360
1361 return (0);
1362 }
1363
1364 static int
gicv3_its_bind_intr(device_t dev,struct intr_irqsrc * isrc)1365 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1366 {
1367 struct gicv3_its_irqsrc *girq;
1368
1369 gicv3_its_select_cpu(dev, isrc);
1370
1371 girq = (struct gicv3_its_irqsrc *)isrc;
1372 its_cmd_movi(dev, girq);
1373 return (0);
1374 }
1375
1376 static int
gicv3_its_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)1377 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
1378 struct intr_irqsrc **isrcp)
1379 {
1380
1381 /*
1382 * This should never happen, we only call this function to map
1383 * interrupts found before the controller driver is ready.
1384 */
1385 panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
1386 }
1387
1388 static int
gicv3_its_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)1389 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1390 struct resource *res, struct intr_map_data *data)
1391 {
1392
1393 /* Bind the interrupt to a CPU */
1394 gicv3_its_bind_intr(dev, isrc);
1395
1396 return (0);
1397 }
1398
1399 #ifdef SMP
1400 static void
gicv3_its_init_secondary(device_t dev,uint32_t rootnum)1401 gicv3_its_init_secondary(device_t dev, uint32_t rootnum)
1402 {
1403 struct gicv3_its_softc *sc;
1404
1405 sc = device_get_softc(dev);
1406
1407 /*
1408 * This is fatal as otherwise we may bind interrupts to this CPU.
1409 * We need a way to tell the interrupt framework to only bind to a
1410 * subset of given CPUs when it performs the shuffle.
1411 */
1412 if (its_init_cpu(dev, sc) != 0)
1413 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
1414 PCPU_GET(cpuid));
1415 }
1416 #endif
1417
1418 static uint32_t
its_get_devid(device_t pci_dev)1419 its_get_devid(device_t pci_dev)
1420 {
1421 uintptr_t id;
1422
1423 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
1424 panic("%s: %s: Unable to get the MSI DeviceID", __func__,
1425 device_get_nameunit(pci_dev));
1426
1427 return (id);
1428 }
1429
1430 static struct its_dev *
its_device_find(device_t dev,device_t child)1431 its_device_find(device_t dev, device_t child)
1432 {
1433 struct gicv3_its_softc *sc;
1434 struct its_dev *its_dev = NULL;
1435
1436 sc = device_get_softc(dev);
1437
1438 mtx_lock_spin(&sc->sc_its_dev_lock);
1439 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
1440 if (its_dev->pci_dev == child)
1441 break;
1442 }
1443 mtx_unlock_spin(&sc->sc_its_dev_lock);
1444
1445 return (its_dev);
1446 }
1447
1448 static bool
its_device_alloc(struct gicv3_its_softc * sc,int devid)1449 its_device_alloc(struct gicv3_its_softc *sc, int devid)
1450 {
1451 struct its_ptable *ptable;
1452 void *l2_table;
1453 uint64_t *table;
1454 uint32_t index;
1455 bool shareable;
1456
1457 /* No device table */
1458 if (sc->sc_dev_table_idx < 0) {
1459 if (devid >= (1 << sc->sc_devbits)) {
1460 if (bootverbose) {
1461 device_printf(sc->dev,
1462 "%s: Device out of range for hardware "
1463 "(%x >= %x)\n", __func__, devid,
1464 1 << sc->sc_devbits);
1465 }
1466 return (false);
1467 }
1468 return (true);
1469 }
1470
1471 ptable = &sc->sc_its_ptab[sc->sc_dev_table_idx];
1472 /* Check the devid is within the table limit */
1473 if (!ptable->ptab_indirect) {
1474 if (devid >= ptable->ptab_l1_nidents) {
1475 if (bootverbose) {
1476 device_printf(sc->dev,
1477 "%s: Device out of range for table "
1478 "(%x >= %x)\n", __func__, devid,
1479 ptable->ptab_l1_nidents);
1480 }
1481 return (false);
1482 }
1483
1484 return (true);
1485 }
1486
1487 /* Check the devid is within the allocated range */
1488 index = devid / ptable->ptab_l2_nidents;
1489 if (index >= ptable->ptab_l1_nidents) {
1490 if (bootverbose) {
1491 device_printf(sc->dev,
1492 "%s: Index out of range for table (%x >= %x)\n",
1493 __func__, index, ptable->ptab_l1_nidents);
1494 }
1495 return (false);
1496 }
1497
1498 table = (uint64_t *)ptable->ptab_vaddr;
1499 /* We have an second level table */
1500 if ((table[index] & GITS_BASER_VALID) != 0)
1501 return (true);
1502
1503 shareable = true;
1504 if ((ptable->ptab_share & GITS_BASER_SHARE_MASK) == GITS_BASER_SHARE_NS)
1505 shareable = false;
1506
1507 l2_table = contigmalloc_domainset(ptable->ptab_l2_size,
1508 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0,
1509 gicv3_its_limit_max_addr(sc, (1ul << 48) - 1),
1510 ptable->ptab_page_size, 0);
1511
1512 if (!shareable)
1513 cpu_dcache_wb_range(l2_table, ptable->ptab_l2_size);
1514
1515 table[index] = vtophys(l2_table) | GITS_BASER_VALID;
1516 if (!shareable)
1517 cpu_dcache_wb_range(&table[index], sizeof(table[index]));
1518
1519 dsb(sy);
1520 return (true);
1521 }
1522
1523 static struct its_dev *
its_device_get(device_t dev,device_t child,u_int nvecs)1524 its_device_get(device_t dev, device_t child, u_int nvecs)
1525 {
1526 struct gicv3_its_softc *sc;
1527 struct its_dev *its_dev;
1528 vmem_addr_t irq_base;
1529 size_t esize, itt_size;
1530
1531 sc = device_get_softc(dev);
1532
1533 its_dev = its_device_find(dev, child);
1534 if (its_dev != NULL)
1535 return (its_dev);
1536
1537 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1538 if (its_dev == NULL)
1539 return (NULL);
1540
1541 its_dev->pci_dev = child;
1542 its_dev->devid = its_get_devid(child);
1543
1544 its_dev->lpis.lpi_busy = 0;
1545 its_dev->lpis.lpi_num = nvecs;
1546 its_dev->lpis.lpi_free = nvecs;
1547
1548 if (!its_device_alloc(sc, its_dev->devid)) {
1549 free(its_dev, M_GICV3_ITS);
1550 return (NULL);
1551 }
1552
1553 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1554 &irq_base) != 0) {
1555 free(its_dev, M_GICV3_ITS);
1556 return (NULL);
1557 }
1558 its_dev->lpis.lpi_base = irq_base;
1559
1560 /* Get ITT entry size */
1561 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1562
1563 /*
1564 * Allocate ITT for this device.
1565 * PA has to be 256 B aligned. At least two entries for device.
1566 */
1567 itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1568 its_dev->itt = contigmalloc_domainset(itt_size,
1569 M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0,
1570 gicv3_its_limit_max_addr(sc, LPI_INT_TRANS_TAB_MAX_ADDR),
1571 LPI_INT_TRANS_TAB_ALIGN, 0);
1572 if (its_dev->itt == NULL) {
1573 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1574 free(its_dev, M_GICV3_ITS);
1575 return (NULL);
1576 }
1577
1578 /* Make sure device sees zeroed ITT. */
1579 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0)
1580 cpu_dcache_wb_range(its_dev->itt, itt_size);
1581
1582 mtx_lock_spin(&sc->sc_its_dev_lock);
1583 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1584 mtx_unlock_spin(&sc->sc_its_dev_lock);
1585
1586 /* Map device to its ITT */
1587 its_cmd_mapd(dev, its_dev, 1);
1588
1589 return (its_dev);
1590 }
1591
1592 static void
its_device_release(device_t dev,struct its_dev * its_dev)1593 its_device_release(device_t dev, struct its_dev *its_dev)
1594 {
1595 struct gicv3_its_softc *sc;
1596
1597 KASSERT(its_dev->lpis.lpi_busy == 0,
1598 ("its_device_release: Trying to release an inuse ITS device"));
1599
1600 /* Unmap device in ITS */
1601 its_cmd_mapd(dev, its_dev, 0);
1602
1603 sc = device_get_softc(dev);
1604
1605 /* Remove the device from the list of devices */
1606 mtx_lock_spin(&sc->sc_its_dev_lock);
1607 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1608 mtx_unlock_spin(&sc->sc_its_dev_lock);
1609
1610 /* Free ITT */
1611 KASSERT(its_dev->itt != NULL, ("Invalid ITT in valid ITS device"));
1612 free(its_dev->itt, M_GICV3_ITS);
1613
1614 /* Free the IRQ allocation */
1615 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1616 its_dev->lpis.lpi_num);
1617
1618 free(its_dev, M_GICV3_ITS);
1619 }
1620
1621 static struct gicv3_its_irqsrc *
gicv3_its_alloc_irqsrc(device_t dev,struct gicv3_its_softc * sc,u_int irq)1622 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq)
1623 {
1624 struct gicv3_its_irqsrc *girq = NULL;
1625
1626 KASSERT(sc->sc_irqs[irq] == NULL,
1627 ("%s: Interrupt %u already allocated", __func__, irq));
1628 mtx_lock_spin(&sc->sc_its_dev_lock);
1629 if (!TAILQ_EMPTY(&sc->sc_free_irqs)) {
1630 girq = TAILQ_FIRST(&sc->sc_free_irqs);
1631 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link);
1632 }
1633 mtx_unlock_spin(&sc->sc_its_dev_lock);
1634 if (girq == NULL) {
1635 girq = malloc(sizeof(*girq), M_GICV3_ITS,
1636 M_NOWAIT | M_ZERO);
1637 if (girq == NULL)
1638 return (NULL);
1639 girq->gi_id = -1;
1640 if (intr_isrc_register(&girq->gi_isrc, dev, 0,
1641 "%s,%u", device_get_nameunit(dev), irq) != 0) {
1642 free(girq, M_GICV3_ITS);
1643 return (NULL);
1644 }
1645 }
1646 girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI;
1647 sc->sc_irqs[irq] = girq;
1648
1649 return (girq);
1650 }
1651
1652 static void
gicv3_its_release_irqsrc(struct gicv3_its_softc * sc,struct gicv3_its_irqsrc * girq)1653 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc,
1654 struct gicv3_its_irqsrc *girq)
1655 {
1656 u_int irq;
1657
1658 mtx_assert(&sc->sc_its_dev_lock, MA_OWNED);
1659
1660 irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base;
1661 sc->sc_irqs[irq] = NULL;
1662
1663 girq->gi_id = -1;
1664 girq->gi_its_dev = NULL;
1665 TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link);
1666 }
1667
1668 static int
gicv3_its_alloc_msi(device_t dev,device_t child,int count,int maxcount,device_t * pic,struct intr_irqsrc ** srcs)1669 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1670 device_t *pic, struct intr_irqsrc **srcs)
1671 {
1672 struct gicv3_its_softc *sc;
1673 struct gicv3_its_irqsrc *girq;
1674 struct its_dev *its_dev;
1675 u_int irq;
1676 int i;
1677
1678 its_dev = its_device_get(dev, child, count);
1679 if (its_dev == NULL)
1680 return (ENXIO);
1681
1682 KASSERT(its_dev->lpis.lpi_free >= count,
1683 ("gicv3_its_alloc_msi: No free LPIs"));
1684 sc = device_get_softc(dev);
1685 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1686 its_dev->lpis.lpi_free;
1687
1688 /* Allocate the irqsrc for each MSI */
1689 for (i = 0; i < count; i++, irq++) {
1690 its_dev->lpis.lpi_free--;
1691 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev,
1692 sc, irq);
1693 if (srcs[i] == NULL)
1694 break;
1695 }
1696
1697 /* The allocation failed, release them */
1698 if (i != count) {
1699 mtx_lock_spin(&sc->sc_its_dev_lock);
1700 for (i = 0; i < count; i++) {
1701 girq = (struct gicv3_its_irqsrc *)srcs[i];
1702 if (girq == NULL)
1703 break;
1704 gicv3_its_release_irqsrc(sc, girq);
1705 srcs[i] = NULL;
1706 }
1707 mtx_unlock_spin(&sc->sc_its_dev_lock);
1708 return (ENXIO);
1709 }
1710
1711 /* Finish the allocation now we have all MSI irqsrcs */
1712 for (i = 0; i < count; i++) {
1713 girq = (struct gicv3_its_irqsrc *)srcs[i];
1714 girq->gi_id = i;
1715 girq->gi_its_dev = its_dev;
1716
1717 /* Map the message to the given IRQ */
1718 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1719 its_cmd_mapti(dev, girq);
1720 }
1721 its_dev->lpis.lpi_busy += count;
1722 *pic = dev;
1723
1724 return (0);
1725 }
1726
1727 static int
gicv3_its_release_msi(device_t dev,device_t child,int count,struct intr_irqsrc ** isrc)1728 gicv3_its_release_msi(device_t dev, device_t child, int count,
1729 struct intr_irqsrc **isrc)
1730 {
1731 struct gicv3_its_softc *sc;
1732 struct gicv3_its_irqsrc *girq;
1733 struct its_dev *its_dev;
1734 int i;
1735
1736 its_dev = its_device_find(dev, child);
1737
1738 KASSERT(its_dev != NULL,
1739 ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1740 "no ITS device"));
1741 KASSERT(its_dev->lpis.lpi_busy >= count,
1742 ("gicv3_its_release_msi: Releasing more interrupts than "
1743 "were allocated: releasing %d, allocated %d", count,
1744 its_dev->lpis.lpi_busy));
1745
1746 sc = device_get_softc(dev);
1747 mtx_lock_spin(&sc->sc_its_dev_lock);
1748 for (i = 0; i < count; i++) {
1749 girq = (struct gicv3_its_irqsrc *)isrc[i];
1750 gicv3_its_release_irqsrc(sc, girq);
1751 }
1752 mtx_unlock_spin(&sc->sc_its_dev_lock);
1753 its_dev->lpis.lpi_busy -= count;
1754
1755 if (its_dev->lpis.lpi_busy == 0)
1756 its_device_release(dev, its_dev);
1757
1758 return (0);
1759 }
1760
1761 static int
gicv3_its_alloc_msix(device_t dev,device_t child,device_t * pic,struct intr_irqsrc ** isrcp)1762 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1763 struct intr_irqsrc **isrcp)
1764 {
1765 struct gicv3_its_softc *sc;
1766 struct gicv3_its_irqsrc *girq;
1767 struct its_dev *its_dev;
1768 u_int nvecs, irq;
1769
1770 nvecs = pci_msix_count(child);
1771 its_dev = its_device_get(dev, child, nvecs);
1772 if (its_dev == NULL)
1773 return (ENXIO);
1774
1775 KASSERT(its_dev->lpis.lpi_free > 0,
1776 ("gicv3_its_alloc_msix: No free LPIs"));
1777 sc = device_get_softc(dev);
1778 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1779 its_dev->lpis.lpi_free;
1780
1781 girq = gicv3_its_alloc_irqsrc(dev, sc, irq);
1782 if (girq == NULL)
1783 return (ENXIO);
1784 girq->gi_id = its_dev->lpis.lpi_busy;
1785 girq->gi_its_dev = its_dev;
1786
1787 its_dev->lpis.lpi_free--;
1788 its_dev->lpis.lpi_busy++;
1789
1790 /* Map the message to the given IRQ */
1791 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1792 its_cmd_mapti(dev, girq);
1793
1794 *pic = dev;
1795 *isrcp = (struct intr_irqsrc *)girq;
1796
1797 return (0);
1798 }
1799
1800 static int
gicv3_its_release_msix(device_t dev,device_t child,struct intr_irqsrc * isrc)1801 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1802 {
1803 struct gicv3_its_softc *sc;
1804 struct gicv3_its_irqsrc *girq;
1805 struct its_dev *its_dev;
1806
1807 its_dev = its_device_find(dev, child);
1808
1809 KASSERT(its_dev != NULL,
1810 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1811 "no ITS device"));
1812 KASSERT(its_dev->lpis.lpi_busy > 0,
1813 ("gicv3_its_release_msix: Releasing more interrupts than "
1814 "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1815
1816 sc = device_get_softc(dev);
1817 girq = (struct gicv3_its_irqsrc *)isrc;
1818 mtx_lock_spin(&sc->sc_its_dev_lock);
1819 gicv3_its_release_irqsrc(sc, girq);
1820 mtx_unlock_spin(&sc->sc_its_dev_lock);
1821 its_dev->lpis.lpi_busy--;
1822
1823 if (its_dev->lpis.lpi_busy == 0)
1824 its_device_release(dev, its_dev);
1825
1826 return (0);
1827 }
1828
1829 static int
gicv3_its_map_msi(device_t dev,device_t child,struct intr_irqsrc * isrc,uint64_t * addr,uint32_t * data)1830 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1831 uint64_t *addr, uint32_t *data)
1832 {
1833 struct gicv3_its_softc *sc;
1834 struct gicv3_its_irqsrc *girq;
1835
1836 sc = device_get_softc(dev);
1837 girq = (struct gicv3_its_irqsrc *)isrc;
1838
1839 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1840 *data = girq->gi_id;
1841
1842 return (0);
1843 }
1844
1845 #ifdef IOMMU
1846 static int
gicv3_iommu_init(device_t dev,device_t child,struct iommu_domain ** domain)1847 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain)
1848 {
1849 struct gicv3_its_softc *sc;
1850 struct iommu_ctx *ctx;
1851 int error;
1852
1853 sc = device_get_softc(dev);
1854 /*
1855 * Get the context. If no context is found then the device isn't
1856 * behind an IOMMU so no setup is needed.
1857 */
1858 ctx = iommu_get_dev_ctx(child);
1859 if (ctx == NULL) {
1860 *domain = NULL;
1861 return (0);
1862 }
1863 /* Map the page containing the GITS_TRANSLATER register. */
1864 error = iommu_map_msi(ctx, PAGE_SIZE, 0,
1865 IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma);
1866 *domain = iommu_get_ctx_domain(ctx);
1867
1868 return (error);
1869 }
1870
1871 static void
gicv3_iommu_deinit(device_t dev,device_t child)1872 gicv3_iommu_deinit(device_t dev, device_t child)
1873 {
1874 struct iommu_ctx *ctx;
1875
1876 ctx = iommu_get_dev_ctx(child);
1877 if (ctx == NULL)
1878 return;
1879
1880 iommu_unmap_msi(ctx);
1881 }
1882 #endif
1883
1884 /*
1885 * Commands handling.
1886 */
1887
1888 static __inline void
cmd_format_command(struct its_cmd * cmd,uint8_t cmd_type)1889 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1890 {
1891 /* Command field: DW0 [7:0] */
1892 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1893 cmd->cmd_dword[0] |= htole64(cmd_type);
1894 }
1895
1896 static __inline void
cmd_format_devid(struct its_cmd * cmd,uint32_t devid)1897 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1898 {
1899 /* Device ID field: DW0 [63:32] */
1900 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1901 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1902 }
1903
1904 static __inline void
cmd_format_size(struct its_cmd * cmd,uint16_t size)1905 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1906 {
1907 /* Size field: DW1 [4:0] */
1908 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1909 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1910 }
1911
1912 static __inline void
cmd_format_id(struct its_cmd * cmd,uint32_t id)1913 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1914 {
1915 /* ID field: DW1 [31:0] */
1916 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1917 cmd->cmd_dword[1] |= htole64(id);
1918 }
1919
1920 static __inline void
cmd_format_pid(struct its_cmd * cmd,uint32_t pid)1921 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1922 {
1923 /* Physical ID field: DW1 [63:32] */
1924 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1925 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1926 }
1927
1928 static __inline void
cmd_format_col(struct its_cmd * cmd,uint16_t col_id)1929 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1930 {
1931 /* Collection field: DW2 [16:0] */
1932 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1933 cmd->cmd_dword[2] |= htole64(col_id);
1934 }
1935
1936 static __inline void
cmd_format_target(struct its_cmd * cmd,uint64_t target)1937 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1938 {
1939 /* Target Address field: DW2 [47:16] */
1940 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1941 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1942 }
1943
1944 static __inline void
cmd_format_itt(struct its_cmd * cmd,uint64_t itt)1945 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1946 {
1947 /* ITT Address field: DW2 [47:8] */
1948 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1949 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1950 }
1951
1952 static __inline void
cmd_format_valid(struct its_cmd * cmd,uint8_t valid)1953 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1954 {
1955 /* Valid field: DW2 [63] */
1956 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1957 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1958 }
1959
1960 static inline bool
its_cmd_queue_full(struct gicv3_its_softc * sc)1961 its_cmd_queue_full(struct gicv3_its_softc *sc)
1962 {
1963 size_t read_idx, next_write_idx;
1964
1965 /* Get the index of the next command */
1966 next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1967 (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1968 /* And the index of the current command being read */
1969 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1970
1971 /*
1972 * The queue is full when the write offset points
1973 * at the command before the current read offset.
1974 */
1975 return (next_write_idx == read_idx);
1976 }
1977
1978 static inline void
its_cmd_sync(struct gicv3_its_softc * sc,struct its_cmd * cmd)1979 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1980 {
1981
1982 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1983 /* Clean D-cache under command. */
1984 cpu_dcache_wb_range(cmd, sizeof(*cmd));
1985 } else {
1986 /* DSB inner shareable, store */
1987 dsb(ishst);
1988 }
1989
1990 }
1991
1992 static inline uint64_t
its_cmd_cwriter_offset(struct gicv3_its_softc * sc,struct its_cmd * cmd)1993 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1994 {
1995 uint64_t off;
1996
1997 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1998
1999 return (off);
2000 }
2001
2002 static void
its_cmd_wait_completion(device_t dev,struct its_cmd * cmd_first,struct its_cmd * cmd_last)2003 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
2004 struct its_cmd *cmd_last)
2005 {
2006 struct gicv3_its_softc *sc;
2007 uint64_t first, last, read;
2008 size_t us_left;
2009
2010 sc = device_get_softc(dev);
2011
2012 /*
2013 * XXX ARM64TODO: This is obviously a significant delay.
2014 * The reason for that is that currently the time frames for
2015 * the command to complete are not known.
2016 */
2017 us_left = 1000000;
2018
2019 first = its_cmd_cwriter_offset(sc, cmd_first);
2020 last = its_cmd_cwriter_offset(sc, cmd_last);
2021
2022 for (;;) {
2023 read = gic_its_read_8(sc, GITS_CREADR);
2024 if (first < last) {
2025 if (read < first || read >= last)
2026 break;
2027 } else if (read < first && read >= last)
2028 break;
2029
2030 if (us_left-- == 0) {
2031 /* This means timeout */
2032 device_printf(dev,
2033 "Timeout while waiting for CMD completion.\n");
2034 return;
2035 }
2036 DELAY(1);
2037 }
2038 }
2039
2040 static struct its_cmd *
its_cmd_alloc_locked(device_t dev)2041 its_cmd_alloc_locked(device_t dev)
2042 {
2043 struct gicv3_its_softc *sc;
2044 struct its_cmd *cmd;
2045 size_t us_left;
2046
2047 sc = device_get_softc(dev);
2048
2049 /*
2050 * XXX ARM64TODO: This is obviously a significant delay.
2051 * The reason for that is that currently the time frames for
2052 * the command to complete (and therefore free the descriptor)
2053 * are not known.
2054 */
2055 us_left = 1000000;
2056
2057 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
2058 while (its_cmd_queue_full(sc)) {
2059 if (us_left-- == 0) {
2060 /* Timeout while waiting for free command */
2061 device_printf(dev,
2062 "Timeout while waiting for free command\n");
2063 return (NULL);
2064 }
2065 DELAY(1);
2066 }
2067
2068 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
2069 sc->sc_its_cmd_next_idx++;
2070 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
2071
2072 return (cmd);
2073 }
2074
2075 static uint64_t
its_cmd_prepare(struct its_cmd * cmd,struct its_cmd_desc * desc)2076 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
2077 {
2078 uint64_t target;
2079 uint8_t cmd_type;
2080 u_int size;
2081
2082 cmd_type = desc->cmd_type;
2083 target = ITS_TARGET_NONE;
2084
2085 switch (cmd_type) {
2086 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */
2087 target = desc->cmd_desc_movi.col->col_target;
2088 cmd_format_command(cmd, ITS_CMD_MOVI);
2089 cmd_format_id(cmd, desc->cmd_desc_movi.id);
2090 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
2091 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
2092 break;
2093 case ITS_CMD_SYNC: /* Wait for previous commands completion */
2094 target = desc->cmd_desc_sync.col->col_target;
2095 cmd_format_command(cmd, ITS_CMD_SYNC);
2096 cmd_format_target(cmd, target);
2097 break;
2098 case ITS_CMD_MAPD: /* Assign ITT to device */
2099 cmd_format_command(cmd, ITS_CMD_MAPD);
2100 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
2101 /*
2102 * Size describes number of bits to encode interrupt IDs
2103 * supported by the device minus one.
2104 * When V (valid) bit is zero, this field should be written
2105 * as zero.
2106 */
2107 if (desc->cmd_desc_mapd.valid != 0) {
2108 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
2109 size = MAX(1, size) - 1;
2110 } else
2111 size = 0;
2112
2113 cmd_format_size(cmd, size);
2114 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
2115 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
2116 break;
2117 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */
2118 target = desc->cmd_desc_mapc.col->col_target;
2119 cmd_format_command(cmd, ITS_CMD_MAPC);
2120 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
2121 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
2122 cmd_format_target(cmd, target);
2123 break;
2124 case ITS_CMD_MAPTI:
2125 target = desc->cmd_desc_mapvi.col->col_target;
2126 cmd_format_command(cmd, ITS_CMD_MAPTI);
2127 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
2128 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
2129 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
2130 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
2131 break;
2132 case ITS_CMD_MAPI:
2133 target = desc->cmd_desc_mapi.col->col_target;
2134 cmd_format_command(cmd, ITS_CMD_MAPI);
2135 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
2136 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
2137 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
2138 break;
2139 case ITS_CMD_INV:
2140 target = desc->cmd_desc_inv.col->col_target;
2141 cmd_format_command(cmd, ITS_CMD_INV);
2142 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
2143 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
2144 break;
2145 case ITS_CMD_INVALL:
2146 cmd_format_command(cmd, ITS_CMD_INVALL);
2147 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
2148 break;
2149 default:
2150 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
2151 }
2152
2153 return (target);
2154 }
2155
2156 static int
its_cmd_send(device_t dev,struct its_cmd_desc * desc)2157 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
2158 {
2159 struct gicv3_its_softc *sc;
2160 struct its_cmd *cmd, *cmd_sync, *cmd_write;
2161 struct its_col col_sync;
2162 struct its_cmd_desc desc_sync;
2163 uint64_t target, cwriter;
2164
2165 sc = device_get_softc(dev);
2166 mtx_lock_spin(&sc->sc_its_cmd_lock);
2167 cmd = its_cmd_alloc_locked(dev);
2168 if (cmd == NULL) {
2169 device_printf(dev, "could not allocate ITS command\n");
2170 mtx_unlock_spin(&sc->sc_its_cmd_lock);
2171 return (EBUSY);
2172 }
2173
2174 target = its_cmd_prepare(cmd, desc);
2175 its_cmd_sync(sc, cmd);
2176
2177 if (target != ITS_TARGET_NONE) {
2178 cmd_sync = its_cmd_alloc_locked(dev);
2179 if (cmd_sync != NULL) {
2180 desc_sync.cmd_type = ITS_CMD_SYNC;
2181 col_sync.col_target = target;
2182 desc_sync.cmd_desc_sync.col = &col_sync;
2183 its_cmd_prepare(cmd_sync, &desc_sync);
2184 its_cmd_sync(sc, cmd_sync);
2185 }
2186 }
2187
2188 /* Update GITS_CWRITER */
2189 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
2190 gic_its_write_8(sc, GITS_CWRITER, cwriter);
2191 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
2192 mtx_unlock_spin(&sc->sc_its_cmd_lock);
2193
2194 its_cmd_wait_completion(dev, cmd, cmd_write);
2195
2196 return (0);
2197 }
2198
2199 /* Handlers to send commands */
2200 static void
its_cmd_movi(device_t dev,struct gicv3_its_irqsrc * girq)2201 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
2202 {
2203 struct gicv3_its_softc *sc;
2204 struct its_cmd_desc desc;
2205 struct its_col *col;
2206
2207 sc = device_get_softc(dev);
2208 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2209
2210 desc.cmd_type = ITS_CMD_MOVI;
2211 desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
2212 desc.cmd_desc_movi.col = col;
2213 desc.cmd_desc_movi.id = girq->gi_id;
2214
2215 its_cmd_send(dev, &desc);
2216 }
2217
2218 static void
its_cmd_mapc(device_t dev,struct its_col * col,uint8_t valid)2219 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
2220 {
2221 struct its_cmd_desc desc;
2222
2223 desc.cmd_type = ITS_CMD_MAPC;
2224 desc.cmd_desc_mapc.col = col;
2225 /*
2226 * Valid bit set - map the collection.
2227 * Valid bit cleared - unmap the collection.
2228 */
2229 desc.cmd_desc_mapc.valid = valid;
2230
2231 its_cmd_send(dev, &desc);
2232 }
2233
2234 static void
its_cmd_mapti(device_t dev,struct gicv3_its_irqsrc * girq)2235 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
2236 {
2237 struct gicv3_its_softc *sc;
2238 struct its_cmd_desc desc;
2239 struct its_col *col;
2240 u_int col_id;
2241
2242 sc = device_get_softc(dev);
2243
2244 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
2245 col = sc->sc_its_cols[col_id];
2246
2247 desc.cmd_type = ITS_CMD_MAPTI;
2248 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
2249 desc.cmd_desc_mapvi.col = col;
2250 /* The EventID sent to the device */
2251 desc.cmd_desc_mapvi.id = girq->gi_id;
2252 /* The physical interrupt presented to softeware */
2253 desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI;
2254
2255 its_cmd_send(dev, &desc);
2256 }
2257
2258 static void
its_cmd_mapd(device_t dev,struct its_dev * its_dev,uint8_t valid)2259 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
2260 {
2261 struct its_cmd_desc desc;
2262
2263 desc.cmd_type = ITS_CMD_MAPD;
2264 desc.cmd_desc_mapd.its_dev = its_dev;
2265 desc.cmd_desc_mapd.valid = valid;
2266
2267 its_cmd_send(dev, &desc);
2268 }
2269
2270 static void
its_cmd_inv(device_t dev,struct its_dev * its_dev,struct gicv3_its_irqsrc * girq)2271 its_cmd_inv(device_t dev, struct its_dev *its_dev,
2272 struct gicv3_its_irqsrc *girq)
2273 {
2274 struct gicv3_its_softc *sc;
2275 struct its_cmd_desc desc;
2276 struct its_col *col;
2277
2278 sc = device_get_softc(dev);
2279 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2280
2281 desc.cmd_type = ITS_CMD_INV;
2282 /* The EventID sent to the device */
2283 desc.cmd_desc_inv.pid = girq->gi_id;
2284 desc.cmd_desc_inv.its_dev = its_dev;
2285 desc.cmd_desc_inv.col = col;
2286
2287 its_cmd_send(dev, &desc);
2288 }
2289
2290 static void
its_cmd_invall(device_t dev,struct its_col * col)2291 its_cmd_invall(device_t dev, struct its_col *col)
2292 {
2293 struct its_cmd_desc desc;
2294
2295 desc.cmd_type = ITS_CMD_INVALL;
2296 desc.cmd_desc_invall.col = col;
2297
2298 its_cmd_send(dev, &desc);
2299 }
2300
2301 #ifdef FDT
2302 static device_probe_t gicv3_its_fdt_probe;
2303 static device_attach_t gicv3_its_fdt_attach;
2304
2305 static device_method_t gicv3_its_fdt_methods[] = {
2306 /* Device interface */
2307 DEVMETHOD(device_probe, gicv3_its_fdt_probe),
2308 DEVMETHOD(device_attach, gicv3_its_fdt_attach),
2309
2310 /* End */
2311 DEVMETHOD_END
2312 };
2313
2314 #define its_baseclasses its_fdt_baseclasses
2315 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
2316 sizeof(struct gicv3_its_softc), gicv3_its_driver);
2317 #undef its_baseclasses
2318
2319 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0,
2320 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2321
2322 static int
gicv3_its_fdt_probe(device_t dev)2323 gicv3_its_fdt_probe(device_t dev)
2324 {
2325
2326 if (!ofw_bus_status_okay(dev))
2327 return (ENXIO);
2328
2329 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
2330 return (ENXIO);
2331
2332 if (!gic_get_support_lpis(dev))
2333 return (ENXIO);
2334
2335 device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2336 return (BUS_PROBE_DEFAULT);
2337 }
2338
2339 static int
gicv3_its_fdt_attach(device_t dev)2340 gicv3_its_fdt_attach(device_t dev)
2341 {
2342 struct gicv3_its_softc *sc;
2343 phandle_t xref, node;
2344 int err;
2345
2346 sc = device_get_softc(dev);
2347 sc->dev = dev;
2348 node = ofw_bus_get_node(dev);
2349 err = gicv3_its_attach(dev);
2350 if (err != 0)
2351 return (err);
2352
2353 if (OF_hasprop(node, "dma-noncoherent"))
2354 sc->sc_its_flags |= ITS_FLAGS_FORCE_NOSHAREABLE;
2355 /* Register this device as a interrupt controller */
2356 xref = OF_xref_from_node(node);
2357 sc->sc_pic = intr_pic_register(dev, xref);
2358 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2359 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2360 if (err != 0) {
2361 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2362 return (err);
2363 }
2364
2365 /* Register this device to handle MSI interrupts */
2366 err = intr_msi_register(dev, xref);
2367 if (err != 0) {
2368 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2369 return (err);
2370 }
2371
2372 return (0);
2373 }
2374 #endif
2375
2376 #ifdef DEV_ACPI
2377 static device_probe_t gicv3_its_acpi_probe;
2378 static device_attach_t gicv3_its_acpi_attach;
2379
2380 static device_method_t gicv3_its_acpi_methods[] = {
2381 /* Device interface */
2382 DEVMETHOD(device_probe, gicv3_its_acpi_probe),
2383 DEVMETHOD(device_attach, gicv3_its_acpi_attach),
2384
2385 /* End */
2386 DEVMETHOD_END
2387 };
2388
2389 #define its_baseclasses its_acpi_baseclasses
2390 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
2391 sizeof(struct gicv3_its_softc), gicv3_its_driver);
2392 #undef its_baseclasses
2393
2394 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0,
2395 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2396
2397 static int
gicv3_its_acpi_probe(device_t dev)2398 gicv3_its_acpi_probe(device_t dev)
2399 {
2400
2401 if (gic_get_bus(dev) != GIC_BUS_ACPI)
2402 return (EINVAL);
2403
2404 if (gic_get_hw_rev(dev) < 3)
2405 return (EINVAL);
2406
2407 if (!gic_get_support_lpis(dev))
2408 return (ENXIO);
2409
2410 device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2411 return (BUS_PROBE_DEFAULT);
2412 }
2413
2414 static int
gicv3_its_acpi_attach(device_t dev)2415 gicv3_its_acpi_attach(device_t dev)
2416 {
2417 struct gicv3_its_softc *sc;
2418 struct gic_v3_devinfo *di;
2419 int err;
2420
2421 sc = device_get_softc(dev);
2422 sc->dev = dev;
2423 err = gicv3_its_attach(dev);
2424 if (err != 0)
2425 return (err);
2426
2427 di = device_get_ivars(dev);
2428 sc->sc_pic = intr_pic_register(dev, di->msi_xref);
2429 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2430 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2431 if (err != 0) {
2432 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2433 return (err);
2434 }
2435
2436 /* Register this device to handle MSI interrupts */
2437 err = intr_msi_register(dev, di->msi_xref);
2438 if (err != 0) {
2439 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2440 return (err);
2441 }
2442
2443 return (0);
2444 }
2445 #endif
2446