1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * ARM Generic Interrupt Controller (GIC) v3 support 4 */ 5 6 #include <linux/sizes.h> 7 8 #include "kvm_util.h" 9 #include "processor.h" 10 #include "delay.h" 11 12 #include "gic.h" 13 #include "gic_v3.h" 14 #include "gic_private.h" 15 16 #define GICV3_MAX_CPUS 512 17 18 #define GICD_INT_DEF_PRI 0xa0 19 #define GICD_INT_DEF_PRI_X4 ((GICD_INT_DEF_PRI << 24) |\ 20 (GICD_INT_DEF_PRI << 16) |\ 21 (GICD_INT_DEF_PRI << 8) |\ 22 GICD_INT_DEF_PRI) 23 24 #define ICC_PMR_DEF_PRIO 0xf0 25 26 struct gicv3_data { 27 unsigned int nr_cpus; 28 unsigned int nr_spis; 29 }; 30 31 #define sgi_base_from_redist(redist_base) (redist_base + SZ_64K) 32 #define DIST_BIT (1U << 31) 33 34 enum gicv3_intid_range { 35 SGI_RANGE, 36 PPI_RANGE, 37 SPI_RANGE, 38 INVALID_RANGE, 39 }; 40 41 static struct gicv3_data gicv3_data; 42 43 static void gicv3_gicd_wait_for_rwp(void) 44 { 45 unsigned int count = 100000; /* 1s */ 46 47 while (readl(GICD_BASE_GVA + GICD_CTLR) & GICD_CTLR_RWP) { 48 GUEST_ASSERT(count--); 49 udelay(10); 50 } 51 } 52 53 static inline volatile void *gicr_base_cpu(uint32_t cpu) 54 { 55 /* Align all the redistributors sequentially */ 56 return GICR_BASE_GVA + cpu * SZ_64K * 2; 57 } 58 59 static void gicv3_gicr_wait_for_rwp(uint32_t cpu) 60 { 61 unsigned int count = 100000; /* 1s */ 62 63 while (readl(gicr_base_cpu(cpu) + GICR_CTLR) & GICR_CTLR_RWP) { 64 GUEST_ASSERT(count--); 65 udelay(10); 66 } 67 } 68 69 static void gicv3_wait_for_rwp(uint32_t cpu_or_dist) 70 { 71 if (cpu_or_dist & DIST_BIT) 72 gicv3_gicd_wait_for_rwp(); 73 else 74 gicv3_gicr_wait_for_rwp(cpu_or_dist); 75 } 76 77 static enum gicv3_intid_range get_intid_range(unsigned int intid) 78 { 79 switch (intid) { 80 case 0 ... 15: 81 return SGI_RANGE; 82 case 16 ... 31: 83 return PPI_RANGE; 84 case 32 ... 1019: 85 return SPI_RANGE; 86 } 87 88 /* We should not be reaching here */ 89 GUEST_ASSERT(0); 90 91 return INVALID_RANGE; 92 } 93 94 static uint64_t gicv3_read_iar(void) 95 { 96 uint64_t irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1); 97 98 dsb(sy); 99 return irqstat; 100 } 101 102 static void gicv3_write_eoir(uint32_t irq) 103 { 104 write_sysreg_s(irq, SYS_ICC_EOIR1_EL1); 105 isb(); 106 } 107 108 static void gicv3_write_dir(uint32_t irq) 109 { 110 write_sysreg_s(irq, SYS_ICC_DIR_EL1); 111 isb(); 112 } 113 114 static void gicv3_set_priority_mask(uint64_t mask) 115 { 116 write_sysreg_s(mask, SYS_ICC_PMR_EL1); 117 } 118 119 static void gicv3_set_eoi_split(bool split) 120 { 121 uint32_t val; 122 123 /* 124 * All other fields are read-only, so no need to read CTLR first. In 125 * fact, the kernel does the same. 126 */ 127 val = split ? (1U << 1) : 0; 128 write_sysreg_s(val, SYS_ICC_CTLR_EL1); 129 isb(); 130 } 131 132 uint32_t gicv3_reg_readl(uint32_t cpu_or_dist, uint64_t offset) 133 { 134 volatile void *base = cpu_or_dist & DIST_BIT ? GICD_BASE_GVA 135 : sgi_base_from_redist(gicr_base_cpu(cpu_or_dist)); 136 return readl(base + offset); 137 } 138 139 void gicv3_reg_writel(uint32_t cpu_or_dist, uint64_t offset, uint32_t reg_val) 140 { 141 volatile void *base = cpu_or_dist & DIST_BIT ? GICD_BASE_GVA 142 : sgi_base_from_redist(gicr_base_cpu(cpu_or_dist)); 143 writel(reg_val, base + offset); 144 } 145 146 uint32_t gicv3_getl_fields(uint32_t cpu_or_dist, uint64_t offset, uint32_t mask) 147 { 148 return gicv3_reg_readl(cpu_or_dist, offset) & mask; 149 } 150 151 void gicv3_setl_fields(uint32_t cpu_or_dist, uint64_t offset, 152 uint32_t mask, uint32_t reg_val) 153 { 154 uint32_t tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask; 155 156 tmp |= (reg_val & mask); 157 gicv3_reg_writel(cpu_or_dist, offset, tmp); 158 } 159 160 /* 161 * We use a single offset for the distributor and redistributor maps as they 162 * have the same value in both. The only exceptions are registers that only 163 * exist in one and not the other, like GICR_WAKER that doesn't exist in the 164 * distributor map. Such registers are conveniently marked as reserved in the 165 * map that doesn't implement it; like GICR_WAKER's offset of 0x0014 being 166 * marked as "Reserved" in the Distributor map. 167 */ 168 static void gicv3_access_reg(uint32_t intid, uint64_t offset, 169 uint32_t reg_bits, uint32_t bits_per_field, 170 bool write, uint32_t *val) 171 { 172 uint32_t cpu = guest_get_vcpuid(); 173 enum gicv3_intid_range intid_range = get_intid_range(intid); 174 uint32_t fields_per_reg, index, mask, shift; 175 uint32_t cpu_or_dist; 176 177 GUEST_ASSERT(bits_per_field <= reg_bits); 178 GUEST_ASSERT(!write || *val < (1U << bits_per_field)); 179 /* 180 * This function does not support 64 bit accesses. Just asserting here 181 * until we implement readq/writeq. 182 */ 183 GUEST_ASSERT(reg_bits == 32); 184 185 fields_per_reg = reg_bits / bits_per_field; 186 index = intid % fields_per_reg; 187 shift = index * bits_per_field; 188 mask = ((1U << bits_per_field) - 1) << shift; 189 190 /* Set offset to the actual register holding intid's config. */ 191 offset += (intid / fields_per_reg) * (reg_bits / 8); 192 193 cpu_or_dist = (intid_range == SPI_RANGE) ? DIST_BIT : cpu; 194 195 if (write) 196 gicv3_setl_fields(cpu_or_dist, offset, mask, *val << shift); 197 *val = gicv3_getl_fields(cpu_or_dist, offset, mask) >> shift; 198 } 199 200 static void gicv3_write_reg(uint32_t intid, uint64_t offset, 201 uint32_t reg_bits, uint32_t bits_per_field, uint32_t val) 202 { 203 gicv3_access_reg(intid, offset, reg_bits, 204 bits_per_field, true, &val); 205 } 206 207 static uint32_t gicv3_read_reg(uint32_t intid, uint64_t offset, 208 uint32_t reg_bits, uint32_t bits_per_field) 209 { 210 uint32_t val; 211 212 gicv3_access_reg(intid, offset, reg_bits, 213 bits_per_field, false, &val); 214 return val; 215 } 216 217 static void gicv3_set_priority(uint32_t intid, uint32_t prio) 218 { 219 gicv3_write_reg(intid, GICD_IPRIORITYR, 32, 8, prio); 220 } 221 222 /* Sets the intid to be level-sensitive or edge-triggered. */ 223 static void gicv3_irq_set_config(uint32_t intid, bool is_edge) 224 { 225 uint32_t val; 226 227 /* N/A for private interrupts. */ 228 GUEST_ASSERT(get_intid_range(intid) == SPI_RANGE); 229 val = is_edge ? 2 : 0; 230 gicv3_write_reg(intid, GICD_ICFGR, 32, 2, val); 231 } 232 233 static void gicv3_irq_enable(uint32_t intid) 234 { 235 bool is_spi = get_intid_range(intid) == SPI_RANGE; 236 uint32_t cpu = guest_get_vcpuid(); 237 238 gicv3_write_reg(intid, GICD_ISENABLER, 32, 1, 1); 239 gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu); 240 } 241 242 static void gicv3_irq_disable(uint32_t intid) 243 { 244 bool is_spi = get_intid_range(intid) == SPI_RANGE; 245 uint32_t cpu = guest_get_vcpuid(); 246 247 gicv3_write_reg(intid, GICD_ICENABLER, 32, 1, 1); 248 gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu); 249 } 250 251 static void gicv3_irq_set_active(uint32_t intid) 252 { 253 gicv3_write_reg(intid, GICD_ISACTIVER, 32, 1, 1); 254 } 255 256 static void gicv3_irq_clear_active(uint32_t intid) 257 { 258 gicv3_write_reg(intid, GICD_ICACTIVER, 32, 1, 1); 259 } 260 261 static bool gicv3_irq_get_active(uint32_t intid) 262 { 263 return gicv3_read_reg(intid, GICD_ISACTIVER, 32, 1); 264 } 265 266 static void gicv3_irq_set_pending(uint32_t intid) 267 { 268 gicv3_write_reg(intid, GICD_ISPENDR, 32, 1, 1); 269 } 270 271 static void gicv3_irq_clear_pending(uint32_t intid) 272 { 273 gicv3_write_reg(intid, GICD_ICPENDR, 32, 1, 1); 274 } 275 276 static bool gicv3_irq_get_pending(uint32_t intid) 277 { 278 return gicv3_read_reg(intid, GICD_ISPENDR, 32, 1); 279 } 280 281 static void gicv3_enable_redist(volatile void *redist_base) 282 { 283 uint32_t val = readl(redist_base + GICR_WAKER); 284 unsigned int count = 100000; /* 1s */ 285 286 val &= ~GICR_WAKER_ProcessorSleep; 287 writel(val, redist_base + GICR_WAKER); 288 289 /* Wait until the processor is 'active' */ 290 while (readl(redist_base + GICR_WAKER) & GICR_WAKER_ChildrenAsleep) { 291 GUEST_ASSERT(count--); 292 udelay(10); 293 } 294 } 295 296 static void gicv3_cpu_init(unsigned int cpu) 297 { 298 volatile void *sgi_base; 299 unsigned int i; 300 volatile void *redist_base_cpu; 301 302 GUEST_ASSERT(cpu < gicv3_data.nr_cpus); 303 304 redist_base_cpu = gicr_base_cpu(cpu); 305 sgi_base = sgi_base_from_redist(redist_base_cpu); 306 307 gicv3_enable_redist(redist_base_cpu); 308 309 /* 310 * Mark all the SGI and PPI interrupts as non-secure Group-1. 311 * Also, deactivate and disable them. 312 */ 313 writel(~0, sgi_base + GICR_IGROUPR0); 314 writel(~0, sgi_base + GICR_ICACTIVER0); 315 writel(~0, sgi_base + GICR_ICENABLER0); 316 317 /* Set a default priority for all the SGIs and PPIs */ 318 for (i = 0; i < 32; i += 4) 319 writel(GICD_INT_DEF_PRI_X4, 320 sgi_base + GICR_IPRIORITYR0 + i); 321 322 gicv3_gicr_wait_for_rwp(cpu); 323 324 /* Enable the GIC system register (ICC_*) access */ 325 write_sysreg_s(read_sysreg_s(SYS_ICC_SRE_EL1) | ICC_SRE_EL1_SRE, 326 SYS_ICC_SRE_EL1); 327 328 /* Set a default priority threshold */ 329 write_sysreg_s(ICC_PMR_DEF_PRIO, SYS_ICC_PMR_EL1); 330 331 /* Enable non-secure Group-1 interrupts */ 332 write_sysreg_s(ICC_IGRPEN1_EL1_MASK, SYS_ICC_IGRPEN1_EL1); 333 } 334 335 static void gicv3_dist_init(void) 336 { 337 unsigned int i; 338 339 /* Disable the distributor until we set things up */ 340 writel(0, GICD_BASE_GVA + GICD_CTLR); 341 gicv3_gicd_wait_for_rwp(); 342 343 /* 344 * Mark all the SPI interrupts as non-secure Group-1. 345 * Also, deactivate and disable them. 346 */ 347 for (i = 32; i < gicv3_data.nr_spis; i += 32) { 348 writel(~0, GICD_BASE_GVA + GICD_IGROUPR + i / 8); 349 writel(~0, GICD_BASE_GVA + GICD_ICACTIVER + i / 8); 350 writel(~0, GICD_BASE_GVA + GICD_ICENABLER + i / 8); 351 } 352 353 /* Set a default priority for all the SPIs */ 354 for (i = 32; i < gicv3_data.nr_spis; i += 4) 355 writel(GICD_INT_DEF_PRI_X4, 356 GICD_BASE_GVA + GICD_IPRIORITYR + i); 357 358 /* Wait for the settings to sync-in */ 359 gicv3_gicd_wait_for_rwp(); 360 361 /* Finally, enable the distributor globally with ARE */ 362 writel(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | 363 GICD_CTLR_ENABLE_G1, GICD_BASE_GVA + GICD_CTLR); 364 gicv3_gicd_wait_for_rwp(); 365 } 366 367 static void gicv3_init(unsigned int nr_cpus) 368 { 369 GUEST_ASSERT(nr_cpus <= GICV3_MAX_CPUS); 370 371 gicv3_data.nr_cpus = nr_cpus; 372 gicv3_data.nr_spis = GICD_TYPER_SPIS( 373 readl(GICD_BASE_GVA + GICD_TYPER)); 374 if (gicv3_data.nr_spis > 1020) 375 gicv3_data.nr_spis = 1020; 376 377 /* 378 * Initialize only the distributor for now. 379 * The redistributor and CPU interfaces are initialized 380 * later for every PE. 381 */ 382 gicv3_dist_init(); 383 } 384 385 const struct gic_common_ops gicv3_ops = { 386 .gic_init = gicv3_init, 387 .gic_cpu_init = gicv3_cpu_init, 388 .gic_irq_enable = gicv3_irq_enable, 389 .gic_irq_disable = gicv3_irq_disable, 390 .gic_read_iar = gicv3_read_iar, 391 .gic_write_eoir = gicv3_write_eoir, 392 .gic_write_dir = gicv3_write_dir, 393 .gic_set_priority_mask = gicv3_set_priority_mask, 394 .gic_set_eoi_split = gicv3_set_eoi_split, 395 .gic_set_priority = gicv3_set_priority, 396 .gic_irq_set_active = gicv3_irq_set_active, 397 .gic_irq_clear_active = gicv3_irq_clear_active, 398 .gic_irq_get_active = gicv3_irq_get_active, 399 .gic_irq_set_pending = gicv3_irq_set_pending, 400 .gic_irq_clear_pending = gicv3_irq_clear_pending, 401 .gic_irq_get_pending = gicv3_irq_get_pending, 402 .gic_irq_set_config = gicv3_irq_set_config, 403 }; 404 405 void gic_rdist_enable_lpis(vm_paddr_t cfg_table, size_t cfg_table_size, 406 vm_paddr_t pend_table) 407 { 408 volatile void *rdist_base = gicr_base_cpu(guest_get_vcpuid()); 409 410 u32 ctlr; 411 u64 val; 412 413 val = (cfg_table | 414 GICR_PROPBASER_InnerShareable | 415 GICR_PROPBASER_RaWaWb | 416 ((ilog2(cfg_table_size) - 1) & GICR_PROPBASER_IDBITS_MASK)); 417 writeq_relaxed(val, rdist_base + GICR_PROPBASER); 418 419 val = (pend_table | 420 GICR_PENDBASER_InnerShareable | 421 GICR_PENDBASER_RaWaWb); 422 writeq_relaxed(val, rdist_base + GICR_PENDBASER); 423 424 ctlr = readl_relaxed(rdist_base + GICR_CTLR); 425 ctlr |= GICR_CTLR_ENABLE_LPIS; 426 writel_relaxed(ctlr, rdist_base + GICR_CTLR); 427 } 428