1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012-2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <hyp/adjust_pc.h> 8 9 #include <linux/compiler.h> 10 #include <linux/irqchip/arm-gic-v3.h> 11 #include <linux/kvm_host.h> 12 13 #include <asm/kvm_emulate.h> 14 #include <asm/kvm_hyp.h> 15 #include <asm/kvm_mmu.h> 16 17 #define vtr_to_max_lr_idx(v) ((v) & 0xf) 18 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1) 19 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5)) 20 21 u64 __gic_v3_get_lr(unsigned int lr) 22 { 23 switch (lr & 0xf) { 24 case 0: 25 return read_gicreg(ICH_LR0_EL2); 26 case 1: 27 return read_gicreg(ICH_LR1_EL2); 28 case 2: 29 return read_gicreg(ICH_LR2_EL2); 30 case 3: 31 return read_gicreg(ICH_LR3_EL2); 32 case 4: 33 return read_gicreg(ICH_LR4_EL2); 34 case 5: 35 return read_gicreg(ICH_LR5_EL2); 36 case 6: 37 return read_gicreg(ICH_LR6_EL2); 38 case 7: 39 return read_gicreg(ICH_LR7_EL2); 40 case 8: 41 return read_gicreg(ICH_LR8_EL2); 42 case 9: 43 return read_gicreg(ICH_LR9_EL2); 44 case 10: 45 return read_gicreg(ICH_LR10_EL2); 46 case 11: 47 return read_gicreg(ICH_LR11_EL2); 48 case 12: 49 return read_gicreg(ICH_LR12_EL2); 50 case 13: 51 return read_gicreg(ICH_LR13_EL2); 52 case 14: 53 return read_gicreg(ICH_LR14_EL2); 54 case 15: 55 return read_gicreg(ICH_LR15_EL2); 56 } 57 58 unreachable(); 59 } 60 61 static void __gic_v3_set_lr(u64 val, int lr) 62 { 63 switch (lr & 0xf) { 64 case 0: 65 write_gicreg(val, ICH_LR0_EL2); 66 break; 67 case 1: 68 write_gicreg(val, ICH_LR1_EL2); 69 break; 70 case 2: 71 write_gicreg(val, ICH_LR2_EL2); 72 break; 73 case 3: 74 write_gicreg(val, ICH_LR3_EL2); 75 break; 76 case 4: 77 write_gicreg(val, ICH_LR4_EL2); 78 break; 79 case 5: 80 write_gicreg(val, ICH_LR5_EL2); 81 break; 82 case 6: 83 write_gicreg(val, ICH_LR6_EL2); 84 break; 85 case 7: 86 write_gicreg(val, ICH_LR7_EL2); 87 break; 88 case 8: 89 write_gicreg(val, ICH_LR8_EL2); 90 break; 91 case 9: 92 write_gicreg(val, ICH_LR9_EL2); 93 break; 94 case 10: 95 write_gicreg(val, ICH_LR10_EL2); 96 break; 97 case 11: 98 write_gicreg(val, ICH_LR11_EL2); 99 break; 100 case 12: 101 write_gicreg(val, ICH_LR12_EL2); 102 break; 103 case 13: 104 write_gicreg(val, ICH_LR13_EL2); 105 break; 106 case 14: 107 write_gicreg(val, ICH_LR14_EL2); 108 break; 109 case 15: 110 write_gicreg(val, ICH_LR15_EL2); 111 break; 112 } 113 } 114 115 static void __vgic_v3_write_ap0rn(u32 val, int n) 116 { 117 switch (n) { 118 case 0: 119 write_gicreg(val, ICH_AP0R0_EL2); 120 break; 121 case 1: 122 write_gicreg(val, ICH_AP0R1_EL2); 123 break; 124 case 2: 125 write_gicreg(val, ICH_AP0R2_EL2); 126 break; 127 case 3: 128 write_gicreg(val, ICH_AP0R3_EL2); 129 break; 130 } 131 } 132 133 static void __vgic_v3_write_ap1rn(u32 val, int n) 134 { 135 switch (n) { 136 case 0: 137 write_gicreg(val, ICH_AP1R0_EL2); 138 break; 139 case 1: 140 write_gicreg(val, ICH_AP1R1_EL2); 141 break; 142 case 2: 143 write_gicreg(val, ICH_AP1R2_EL2); 144 break; 145 case 3: 146 write_gicreg(val, ICH_AP1R3_EL2); 147 break; 148 } 149 } 150 151 static u32 __vgic_v3_read_ap0rn(int n) 152 { 153 u32 val; 154 155 switch (n) { 156 case 0: 157 val = read_gicreg(ICH_AP0R0_EL2); 158 break; 159 case 1: 160 val = read_gicreg(ICH_AP0R1_EL2); 161 break; 162 case 2: 163 val = read_gicreg(ICH_AP0R2_EL2); 164 break; 165 case 3: 166 val = read_gicreg(ICH_AP0R3_EL2); 167 break; 168 default: 169 unreachable(); 170 } 171 172 return val; 173 } 174 175 static u32 __vgic_v3_read_ap1rn(int n) 176 { 177 u32 val; 178 179 switch (n) { 180 case 0: 181 val = read_gicreg(ICH_AP1R0_EL2); 182 break; 183 case 1: 184 val = read_gicreg(ICH_AP1R1_EL2); 185 break; 186 case 2: 187 val = read_gicreg(ICH_AP1R2_EL2); 188 break; 189 case 3: 190 val = read_gicreg(ICH_AP1R3_EL2); 191 break; 192 default: 193 unreachable(); 194 } 195 196 return val; 197 } 198 199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if) 200 { 201 u64 used_lrs = cpu_if->used_lrs; 202 203 /* 204 * Make sure stores to the GIC via the memory mapped interface 205 * are now visible to the system register interface when reading the 206 * LRs, and when reading back the VMCR on non-VHE systems. 207 */ 208 if (used_lrs || !has_vhe()) { 209 if (!cpu_if->vgic_sre) { 210 dsb(sy); 211 isb(); 212 } 213 } 214 215 if (used_lrs || cpu_if->its_vpe.its_vm) { 216 int i; 217 u32 elrsr; 218 219 elrsr = read_gicreg(ICH_ELRSR_EL2); 220 221 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EL2_En, ICH_HCR_EL2); 222 223 for (i = 0; i < used_lrs; i++) { 224 if (elrsr & (1 << i)) 225 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE; 226 else 227 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i); 228 229 __gic_v3_set_lr(0, i); 230 } 231 } 232 } 233 234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if) 235 { 236 u64 used_lrs = cpu_if->used_lrs; 237 int i; 238 239 if (used_lrs || cpu_if->its_vpe.its_vm) { 240 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); 241 242 for (i = 0; i < used_lrs; i++) 243 __gic_v3_set_lr(cpu_if->vgic_lr[i], i); 244 } 245 246 /* 247 * Ensure that writes to the LRs, and on non-VHE systems ensure that 248 * the write to the VMCR in __vgic_v3_activate_traps(), will have 249 * reached the (re)distributors. This ensure the guest will read the 250 * correct values from the memory-mapped interface. 251 */ 252 if (used_lrs || !has_vhe()) { 253 if (!cpu_if->vgic_sre) { 254 isb(); 255 dsb(sy); 256 } 257 } 258 } 259 260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if) 261 { 262 /* 263 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a 264 * Group0 interrupt (as generated in GICv2 mode) to be 265 * delivered as a FIQ to the guest, with potentially fatal 266 * consequences. So we must make sure that ICC_SRE_EL1 has 267 * been actually programmed with the value we want before 268 * starting to mess with the rest of the GIC, and VMCR_EL2 in 269 * particular. This logic must be called before 270 * __vgic_v3_restore_state(). 271 * 272 * However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is 273 * provisioned at all. In order to prevent illegal accesses to the 274 * system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1 275 * so that the trap bits can take effect. Yes, we *loves* the GIC. 276 */ 277 if (!(cpu_if->vgic_hcr & ICH_HCR_EL2_En)) { 278 write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1); 279 isb(); 280 } else if (!cpu_if->vgic_sre) { 281 write_gicreg(0, ICC_SRE_EL1); 282 isb(); 283 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2); 284 285 286 if (has_vhe()) { 287 /* 288 * Ensure that the write to the VMCR will have reached 289 * the (re)distributors. This ensure the guest will 290 * read the correct values from the memory-mapped 291 * interface. 292 */ 293 isb(); 294 dsb(sy); 295 } 296 } 297 298 /* 299 * GICv5 BET0 FEAT_GCIE_LEGACY doesn't include ICC_SRE_EL2. This is due 300 * to be relaxed in a future spec release, at which point this in 301 * condition can be dropped. 302 */ 303 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) { 304 /* 305 * Prevent the guest from touching the ICC_SRE_EL1 system 306 * register. Note that this may not have any effect, as 307 * ICC_SRE_EL2.Enable being RAO/WI is a valid implementation. 308 */ 309 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE, 310 ICC_SRE_EL2); 311 } 312 313 /* 314 * If we need to trap system registers, we must write 315 * ICH_HCR_EL2 anyway, even if no interrupts are being 316 * injected. Note that this also applies if we don't expect 317 * any system register access (no vgic at all). 318 */ 319 if (static_branch_unlikely(&vgic_v3_cpuif_trap) || 320 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre) 321 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); 322 } 323 324 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if) 325 { 326 u64 val; 327 328 if (!cpu_if->vgic_sre) { 329 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2); 330 } 331 332 /* 333 * Can be dropped in the future when GICv5 spec is relaxed. See comment 334 * above. 335 */ 336 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) { 337 val = read_gicreg(ICC_SRE_EL2); 338 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2); 339 } 340 341 if (!cpu_if->vgic_sre) { 342 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */ 343 isb(); 344 write_gicreg(1, ICC_SRE_EL1); 345 } 346 347 /* 348 * If we were trapping system registers, we enabled the VGIC even if 349 * no interrupts were being injected, and we disable it again here. 350 */ 351 if (static_branch_unlikely(&vgic_v3_cpuif_trap) || 352 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre) 353 write_gicreg(0, ICH_HCR_EL2); 354 } 355 356 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if) 357 { 358 u64 val; 359 u32 nr_pre_bits; 360 361 val = read_gicreg(ICH_VTR_EL2); 362 nr_pre_bits = vtr_to_nr_pre_bits(val); 363 364 switch (nr_pre_bits) { 365 case 7: 366 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3); 367 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2); 368 fallthrough; 369 case 6: 370 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1); 371 fallthrough; 372 default: 373 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0); 374 } 375 376 switch (nr_pre_bits) { 377 case 7: 378 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3); 379 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2); 380 fallthrough; 381 case 6: 382 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1); 383 fallthrough; 384 default: 385 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0); 386 } 387 } 388 389 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) 390 { 391 u64 val; 392 u32 nr_pre_bits; 393 394 val = read_gicreg(ICH_VTR_EL2); 395 nr_pre_bits = vtr_to_nr_pre_bits(val); 396 397 switch (nr_pre_bits) { 398 case 7: 399 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3); 400 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2); 401 fallthrough; 402 case 6: 403 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1); 404 fallthrough; 405 default: 406 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0); 407 } 408 409 switch (nr_pre_bits) { 410 case 7: 411 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3); 412 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2); 413 fallthrough; 414 case 6: 415 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1); 416 fallthrough; 417 default: 418 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0); 419 } 420 } 421 422 void __vgic_v3_init_lrs(void) 423 { 424 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2)); 425 int i; 426 427 for (i = 0; i <= max_lr_idx; i++) 428 __gic_v3_set_lr(0, i); 429 } 430 431 /* 432 * Return the GIC CPU configuration: 433 * - [31:0] ICH_VTR_EL2 434 * - [62:32] RES0 435 * - [63] MMIO (GICv2) capable 436 */ 437 u64 __vgic_v3_get_gic_config(void) 438 { 439 u64 val, sre; 440 unsigned long flags = 0; 441 442 /* 443 * In compat mode, we cannot access ICC_SRE_EL1 at any EL 444 * other than EL1 itself; just return the 445 * ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5 446 * system, so we first check if we have GICv5 support. 447 */ 448 if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) 449 return read_gicreg(ICH_VTR_EL2); 450 451 sre = read_gicreg(ICC_SRE_EL1); 452 /* 453 * To check whether we have a MMIO-based (GICv2 compatible) 454 * CPU interface, we need to disable the system register 455 * view. 456 * 457 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates 458 * that to be able to set ICC_SRE_EL1.SRE to 0, all the 459 * interrupt overrides must be set. You've got to love this. 460 * 461 * As we always run VHE with HCR_xMO set, no extra xMO 462 * manipulation is required in that case. 463 * 464 * To safely disable SRE, we have to prevent any interrupt 465 * from firing (which would be deadly). This only makes sense 466 * on VHE, as interrupts are already masked for nVHE as part 467 * of the exception entry to EL2. 468 */ 469 if (has_vhe()) { 470 flags = local_daif_save(); 471 } else { 472 sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO); 473 isb(); 474 } 475 476 write_gicreg(0, ICC_SRE_EL1); 477 isb(); 478 479 val = read_gicreg(ICC_SRE_EL1); 480 481 write_gicreg(sre, ICC_SRE_EL1); 482 isb(); 483 484 if (has_vhe()) { 485 local_daif_restore(flags); 486 } else { 487 sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0); 488 isb(); 489 } 490 491 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); 492 val |= read_gicreg(ICH_VTR_EL2); 493 494 return val; 495 } 496 497 static void __vgic_v3_compat_mode_enable(void) 498 { 499 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) 500 return; 501 502 sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3); 503 /* Wait for V3 to become enabled */ 504 isb(); 505 } 506 507 static u64 __vgic_v3_read_vmcr(void) 508 { 509 return read_gicreg(ICH_VMCR_EL2); 510 } 511 512 static void __vgic_v3_write_vmcr(u32 vmcr) 513 { 514 write_gicreg(vmcr, ICH_VMCR_EL2); 515 } 516 517 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) 518 { 519 __vgic_v3_save_aprs(cpu_if); 520 if (cpu_if->vgic_sre) 521 cpu_if->vgic_vmcr = __vgic_v3_read_vmcr(); 522 } 523 524 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) 525 { 526 __vgic_v3_compat_mode_enable(); 527 528 /* 529 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen 530 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the 531 * VMCR_EL2 save/restore in the world switch. 532 */ 533 if (cpu_if->vgic_sre) 534 __vgic_v3_write_vmcr(cpu_if->vgic_vmcr); 535 __vgic_v3_restore_aprs(cpu_if); 536 } 537 538 static int __vgic_v3_bpr_min(void) 539 { 540 /* See Pseudocode for VPriorityGroup */ 541 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2)); 542 } 543 544 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu) 545 { 546 u64 esr = kvm_vcpu_get_esr(vcpu); 547 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; 548 549 return crm != 8; 550 } 551 552 #define GICv3_IDLE_PRIORITY 0xff 553 554 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr, 555 u64 *lr_val) 556 { 557 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; 558 u8 priority = GICv3_IDLE_PRIORITY; 559 int i, lr = -1; 560 561 for (i = 0; i < used_lrs; i++) { 562 u64 val = __gic_v3_get_lr(i); 563 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 564 565 /* Not pending in the state? */ 566 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT) 567 continue; 568 569 /* Group-0 interrupt, but Group-0 disabled? */ 570 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK)) 571 continue; 572 573 /* Group-1 interrupt, but Group-1 disabled? */ 574 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK)) 575 continue; 576 577 /* Not the highest priority? */ 578 if (lr_prio >= priority) 579 continue; 580 581 /* This is a candidate */ 582 priority = lr_prio; 583 *lr_val = val; 584 lr = i; 585 } 586 587 if (lr == -1) 588 *lr_val = ICC_IAR1_EL1_SPURIOUS; 589 590 return lr; 591 } 592 593 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid, 594 u64 *lr_val) 595 { 596 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; 597 int i; 598 599 for (i = 0; i < used_lrs; i++) { 600 u64 val = __gic_v3_get_lr(i); 601 602 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid && 603 (val & ICH_LR_ACTIVE_BIT)) { 604 *lr_val = val; 605 return i; 606 } 607 } 608 609 *lr_val = ICC_IAR1_EL1_SPURIOUS; 610 return -1; 611 } 612 613 static int __vgic_v3_get_highest_active_priority(void) 614 { 615 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); 616 u32 hap = 0; 617 int i; 618 619 for (i = 0; i < nr_apr_regs; i++) { 620 u32 val; 621 622 /* 623 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers 624 * contain the active priority levels for this VCPU 625 * for the maximum number of supported priority 626 * levels, and we return the full priority level only 627 * if the BPR is programmed to its minimum, otherwise 628 * we return a combination of the priority level and 629 * subpriority, as determined by the setting of the 630 * BPR, but without the full subpriority. 631 */ 632 val = __vgic_v3_read_ap0rn(i); 633 val |= __vgic_v3_read_ap1rn(i); 634 if (!val) { 635 hap += 32; 636 continue; 637 } 638 639 return (hap + __ffs(val)) << __vgic_v3_bpr_min(); 640 } 641 642 return GICv3_IDLE_PRIORITY; 643 } 644 645 static unsigned int __vgic_v3_get_bpr0(u32 vmcr) 646 { 647 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT; 648 } 649 650 static unsigned int __vgic_v3_get_bpr1(u32 vmcr) 651 { 652 unsigned int bpr; 653 654 if (vmcr & ICH_VMCR_CBPR_MASK) { 655 bpr = __vgic_v3_get_bpr0(vmcr); 656 if (bpr < 7) 657 bpr++; 658 } else { 659 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT; 660 } 661 662 return bpr; 663 } 664 665 /* 666 * Convert a priority to a preemption level, taking the relevant BPR 667 * into account by zeroing the sub-priority bits. 668 */ 669 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp) 670 { 671 unsigned int bpr; 672 673 if (!grp) 674 bpr = __vgic_v3_get_bpr0(vmcr) + 1; 675 else 676 bpr = __vgic_v3_get_bpr1(vmcr); 677 678 return pri & (GENMASK(7, 0) << bpr); 679 } 680 681 /* 682 * The priority value is independent of any of the BPR values, so we 683 * normalize it using the minimal BPR value. This guarantees that no 684 * matter what the guest does with its BPR, we can always set/get the 685 * same value of a priority. 686 */ 687 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp) 688 { 689 u8 pre, ap; 690 u32 val; 691 int apr; 692 693 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp); 694 ap = pre >> __vgic_v3_bpr_min(); 695 apr = ap / 32; 696 697 if (!grp) { 698 val = __vgic_v3_read_ap0rn(apr); 699 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr); 700 } else { 701 val = __vgic_v3_read_ap1rn(apr); 702 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr); 703 } 704 } 705 706 static int __vgic_v3_clear_highest_active_priority(void) 707 { 708 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); 709 u32 hap = 0; 710 int i; 711 712 for (i = 0; i < nr_apr_regs; i++) { 713 u32 ap0, ap1; 714 int c0, c1; 715 716 ap0 = __vgic_v3_read_ap0rn(i); 717 ap1 = __vgic_v3_read_ap1rn(i); 718 if (!ap0 && !ap1) { 719 hap += 32; 720 continue; 721 } 722 723 c0 = ap0 ? __ffs(ap0) : 32; 724 c1 = ap1 ? __ffs(ap1) : 32; 725 726 /* Always clear the LSB, which is the highest priority */ 727 if (c0 < c1) { 728 ap0 &= ~BIT(c0); 729 __vgic_v3_write_ap0rn(ap0, i); 730 hap += c0; 731 } else { 732 ap1 &= ~BIT(c1); 733 __vgic_v3_write_ap1rn(ap1, i); 734 hap += c1; 735 } 736 737 /* Rescale to 8 bits of priority */ 738 return hap << __vgic_v3_bpr_min(); 739 } 740 741 return GICv3_IDLE_PRIORITY; 742 } 743 744 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 745 { 746 u64 lr_val; 747 u8 lr_prio, pmr; 748 int lr, grp; 749 750 grp = __vgic_v3_get_group(vcpu); 751 752 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); 753 if (lr < 0) 754 goto spurious; 755 756 if (grp != !!(lr_val & ICH_LR_GROUP)) 757 goto spurious; 758 759 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT; 760 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 761 if (pmr <= lr_prio) 762 goto spurious; 763 764 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp)) 765 goto spurious; 766 767 lr_val &= ~ICH_LR_STATE; 768 lr_val |= ICH_LR_ACTIVE_BIT; 769 __gic_v3_set_lr(lr_val, lr); 770 __vgic_v3_set_active_priority(lr_prio, vmcr, grp); 771 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); 772 return; 773 774 spurious: 775 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS); 776 } 777 778 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val) 779 { 780 lr_val &= ~ICH_LR_ACTIVE_BIT; 781 if (lr_val & ICH_LR_HW) { 782 u32 pid; 783 784 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT; 785 gic_write_dir(pid); 786 } 787 788 __gic_v3_set_lr(lr_val, lr); 789 } 790 791 static void __vgic_v3_bump_eoicount(void) 792 { 793 u32 hcr; 794 795 hcr = read_gicreg(ICH_HCR_EL2); 796 hcr += 1 << ICH_HCR_EL2_EOIcount_SHIFT; 797 write_gicreg(hcr, ICH_HCR_EL2); 798 } 799 800 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 801 { 802 u32 vid = vcpu_get_reg(vcpu, rt); 803 u64 lr_val; 804 int lr; 805 806 /* EOImode == 0, nothing to be done here */ 807 if (!(vmcr & ICH_VMCR_EOIM_MASK)) 808 return; 809 810 /* No deactivate to be performed on an LPI */ 811 if (vid >= VGIC_MIN_LPI) 812 return; 813 814 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); 815 if (lr == -1) { 816 __vgic_v3_bump_eoicount(); 817 return; 818 } 819 820 __vgic_v3_clear_active_lr(lr, lr_val); 821 } 822 823 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 824 { 825 u32 vid = vcpu_get_reg(vcpu, rt); 826 u64 lr_val; 827 u8 lr_prio, act_prio; 828 int lr, grp; 829 830 grp = __vgic_v3_get_group(vcpu); 831 832 /* Drop priority in any case */ 833 act_prio = __vgic_v3_clear_highest_active_priority(); 834 835 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); 836 if (lr == -1) { 837 /* Do not bump EOIcount for LPIs that aren't in the LRs */ 838 if (!(vid >= VGIC_MIN_LPI)) 839 __vgic_v3_bump_eoicount(); 840 return; 841 } 842 843 /* EOImode == 1 and not an LPI, nothing to be done here */ 844 if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI)) 845 return; 846 847 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 848 849 /* If priorities or group do not match, the guest has fscked-up. */ 850 if (grp != !!(lr_val & ICH_LR_GROUP) || 851 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio) 852 return; 853 854 /* Let's now perform the deactivation */ 855 __vgic_v3_clear_active_lr(lr, lr_val); 856 } 857 858 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 859 { 860 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK)); 861 } 862 863 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 864 { 865 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK)); 866 } 867 868 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 869 { 870 u64 val = vcpu_get_reg(vcpu, rt); 871 872 if (val & 1) 873 vmcr |= ICH_VMCR_ENG0_MASK; 874 else 875 vmcr &= ~ICH_VMCR_ENG0_MASK; 876 877 __vgic_v3_write_vmcr(vmcr); 878 } 879 880 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 881 { 882 u64 val = vcpu_get_reg(vcpu, rt); 883 884 if (val & 1) 885 vmcr |= ICH_VMCR_ENG1_MASK; 886 else 887 vmcr &= ~ICH_VMCR_ENG1_MASK; 888 889 __vgic_v3_write_vmcr(vmcr); 890 } 891 892 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 893 { 894 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr)); 895 } 896 897 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 898 { 899 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr)); 900 } 901 902 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 903 { 904 u64 val = vcpu_get_reg(vcpu, rt); 905 u8 bpr_min = __vgic_v3_bpr_min() - 1; 906 907 /* Enforce BPR limiting */ 908 if (val < bpr_min) 909 val = bpr_min; 910 911 val <<= ICH_VMCR_BPR0_SHIFT; 912 val &= ICH_VMCR_BPR0_MASK; 913 vmcr &= ~ICH_VMCR_BPR0_MASK; 914 vmcr |= val; 915 916 __vgic_v3_write_vmcr(vmcr); 917 } 918 919 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 920 { 921 u64 val = vcpu_get_reg(vcpu, rt); 922 u8 bpr_min = __vgic_v3_bpr_min(); 923 924 if (vmcr & ICH_VMCR_CBPR_MASK) 925 return; 926 927 /* Enforce BPR limiting */ 928 if (val < bpr_min) 929 val = bpr_min; 930 931 val <<= ICH_VMCR_BPR1_SHIFT; 932 val &= ICH_VMCR_BPR1_MASK; 933 vmcr &= ~ICH_VMCR_BPR1_MASK; 934 vmcr |= val; 935 936 __vgic_v3_write_vmcr(vmcr); 937 } 938 939 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n) 940 { 941 u32 val; 942 943 if (!__vgic_v3_get_group(vcpu)) 944 val = __vgic_v3_read_ap0rn(n); 945 else 946 val = __vgic_v3_read_ap1rn(n); 947 948 vcpu_set_reg(vcpu, rt, val); 949 } 950 951 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n) 952 { 953 u32 val = vcpu_get_reg(vcpu, rt); 954 955 if (!__vgic_v3_get_group(vcpu)) 956 __vgic_v3_write_ap0rn(val, n); 957 else 958 __vgic_v3_write_ap1rn(val, n); 959 } 960 961 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu, 962 u32 vmcr, int rt) 963 { 964 __vgic_v3_read_apxrn(vcpu, rt, 0); 965 } 966 967 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu, 968 u32 vmcr, int rt) 969 { 970 __vgic_v3_read_apxrn(vcpu, rt, 1); 971 } 972 973 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 974 { 975 __vgic_v3_read_apxrn(vcpu, rt, 2); 976 } 977 978 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 979 { 980 __vgic_v3_read_apxrn(vcpu, rt, 3); 981 } 982 983 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 984 { 985 __vgic_v3_write_apxrn(vcpu, rt, 0); 986 } 987 988 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 989 { 990 __vgic_v3_write_apxrn(vcpu, rt, 1); 991 } 992 993 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 994 { 995 __vgic_v3_write_apxrn(vcpu, rt, 2); 996 } 997 998 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 999 { 1000 __vgic_v3_write_apxrn(vcpu, rt, 3); 1001 } 1002 1003 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1004 { 1005 u64 lr_val; 1006 int lr, lr_grp, grp; 1007 1008 grp = __vgic_v3_get_group(vcpu); 1009 1010 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); 1011 if (lr == -1) 1012 goto spurious; 1013 1014 lr_grp = !!(lr_val & ICH_LR_GROUP); 1015 if (lr_grp != grp) 1016 lr_val = ICC_IAR1_EL1_SPURIOUS; 1017 1018 spurious: 1019 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); 1020 } 1021 1022 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1023 { 1024 vmcr &= ICH_VMCR_PMR_MASK; 1025 vmcr >>= ICH_VMCR_PMR_SHIFT; 1026 vcpu_set_reg(vcpu, rt, vmcr); 1027 } 1028 1029 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1030 { 1031 u32 val = vcpu_get_reg(vcpu, rt); 1032 1033 val <<= ICH_VMCR_PMR_SHIFT; 1034 val &= ICH_VMCR_PMR_MASK; 1035 vmcr &= ~ICH_VMCR_PMR_MASK; 1036 vmcr |= val; 1037 1038 write_gicreg(vmcr, ICH_VMCR_EL2); 1039 } 1040 1041 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1042 { 1043 u32 val = __vgic_v3_get_highest_active_priority(); 1044 vcpu_set_reg(vcpu, rt, val); 1045 } 1046 1047 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1048 { 1049 u32 vtr, val; 1050 1051 vtr = read_gicreg(ICH_VTR_EL2); 1052 /* PRIbits */ 1053 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT; 1054 /* IDbits */ 1055 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT; 1056 /* A3V */ 1057 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT; 1058 /* EOImode */ 1059 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT; 1060 /* CBPR */ 1061 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT; 1062 1063 vcpu_set_reg(vcpu, rt, val); 1064 } 1065 1066 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1067 { 1068 u32 val = vcpu_get_reg(vcpu, rt); 1069 1070 if (val & ICC_CTLR_EL1_CBPR_MASK) 1071 vmcr |= ICH_VMCR_CBPR_MASK; 1072 else 1073 vmcr &= ~ICH_VMCR_CBPR_MASK; 1074 1075 if (val & ICC_CTLR_EL1_EOImode_MASK) 1076 vmcr |= ICH_VMCR_EOIM_MASK; 1077 else 1078 vmcr &= ~ICH_VMCR_EOIM_MASK; 1079 1080 write_gicreg(vmcr, ICH_VMCR_EL2); 1081 } 1082 1083 static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu, 1084 u32 sysreg, bool is_read) 1085 { 1086 u64 ich_hcr; 1087 1088 if (!is_nested_ctxt(vcpu)) 1089 return false; 1090 1091 ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2); 1092 1093 switch (sysreg) { 1094 case SYS_ICC_IGRPEN0_EL1: 1095 if (is_read && 1096 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1)) 1097 return true; 1098 1099 if (!is_read && 1100 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1)) 1101 return true; 1102 1103 fallthrough; 1104 1105 case SYS_ICC_AP0Rn_EL1(0): 1106 case SYS_ICC_AP0Rn_EL1(1): 1107 case SYS_ICC_AP0Rn_EL1(2): 1108 case SYS_ICC_AP0Rn_EL1(3): 1109 case SYS_ICC_BPR0_EL1: 1110 case SYS_ICC_EOIR0_EL1: 1111 case SYS_ICC_HPPIR0_EL1: 1112 case SYS_ICC_IAR0_EL1: 1113 return ich_hcr & ICH_HCR_EL2_TALL0; 1114 1115 case SYS_ICC_IGRPEN1_EL1: 1116 if (is_read && 1117 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1)) 1118 return true; 1119 1120 if (!is_read && 1121 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1)) 1122 return true; 1123 1124 fallthrough; 1125 1126 case SYS_ICC_AP1Rn_EL1(0): 1127 case SYS_ICC_AP1Rn_EL1(1): 1128 case SYS_ICC_AP1Rn_EL1(2): 1129 case SYS_ICC_AP1Rn_EL1(3): 1130 case SYS_ICC_BPR1_EL1: 1131 case SYS_ICC_EOIR1_EL1: 1132 case SYS_ICC_HPPIR1_EL1: 1133 case SYS_ICC_IAR1_EL1: 1134 return ich_hcr & ICH_HCR_EL2_TALL1; 1135 1136 case SYS_ICC_DIR_EL1: 1137 if (ich_hcr & ICH_HCR_EL2_TDIR) 1138 return true; 1139 1140 fallthrough; 1141 1142 case SYS_ICC_RPR_EL1: 1143 case SYS_ICC_CTLR_EL1: 1144 case SYS_ICC_PMR_EL1: 1145 return ich_hcr & ICH_HCR_EL2_TC; 1146 1147 default: 1148 return false; 1149 } 1150 } 1151 1152 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu) 1153 { 1154 int rt; 1155 u64 esr; 1156 u32 vmcr; 1157 void (*fn)(struct kvm_vcpu *, u32, int); 1158 bool is_read; 1159 u32 sysreg; 1160 1161 if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) 1162 return 0; 1163 1164 esr = kvm_vcpu_get_esr(vcpu); 1165 if (vcpu_mode_is_32bit(vcpu)) { 1166 if (!kvm_condition_valid(vcpu)) { 1167 __kvm_skip_instr(vcpu); 1168 return 1; 1169 } 1170 1171 sysreg = esr_cp15_to_sysreg(esr); 1172 } else { 1173 sysreg = esr_sys64_to_sysreg(esr); 1174 } 1175 1176 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ; 1177 1178 if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read)) 1179 return 0; 1180 1181 switch (sysreg) { 1182 case SYS_ICC_IAR0_EL1: 1183 case SYS_ICC_IAR1_EL1: 1184 if (unlikely(!is_read)) 1185 return 0; 1186 fn = __vgic_v3_read_iar; 1187 break; 1188 case SYS_ICC_EOIR0_EL1: 1189 case SYS_ICC_EOIR1_EL1: 1190 if (unlikely(is_read)) 1191 return 0; 1192 fn = __vgic_v3_write_eoir; 1193 break; 1194 case SYS_ICC_IGRPEN1_EL1: 1195 if (is_read) 1196 fn = __vgic_v3_read_igrpen1; 1197 else 1198 fn = __vgic_v3_write_igrpen1; 1199 break; 1200 case SYS_ICC_BPR1_EL1: 1201 if (is_read) 1202 fn = __vgic_v3_read_bpr1; 1203 else 1204 fn = __vgic_v3_write_bpr1; 1205 break; 1206 case SYS_ICC_AP0Rn_EL1(0): 1207 case SYS_ICC_AP1Rn_EL1(0): 1208 if (is_read) 1209 fn = __vgic_v3_read_apxr0; 1210 else 1211 fn = __vgic_v3_write_apxr0; 1212 break; 1213 case SYS_ICC_AP0Rn_EL1(1): 1214 case SYS_ICC_AP1Rn_EL1(1): 1215 if (is_read) 1216 fn = __vgic_v3_read_apxr1; 1217 else 1218 fn = __vgic_v3_write_apxr1; 1219 break; 1220 case SYS_ICC_AP0Rn_EL1(2): 1221 case SYS_ICC_AP1Rn_EL1(2): 1222 if (is_read) 1223 fn = __vgic_v3_read_apxr2; 1224 else 1225 fn = __vgic_v3_write_apxr2; 1226 break; 1227 case SYS_ICC_AP0Rn_EL1(3): 1228 case SYS_ICC_AP1Rn_EL1(3): 1229 if (is_read) 1230 fn = __vgic_v3_read_apxr3; 1231 else 1232 fn = __vgic_v3_write_apxr3; 1233 break; 1234 case SYS_ICC_HPPIR0_EL1: 1235 case SYS_ICC_HPPIR1_EL1: 1236 if (unlikely(!is_read)) 1237 return 0; 1238 fn = __vgic_v3_read_hppir; 1239 break; 1240 case SYS_ICC_IGRPEN0_EL1: 1241 if (is_read) 1242 fn = __vgic_v3_read_igrpen0; 1243 else 1244 fn = __vgic_v3_write_igrpen0; 1245 break; 1246 case SYS_ICC_BPR0_EL1: 1247 if (is_read) 1248 fn = __vgic_v3_read_bpr0; 1249 else 1250 fn = __vgic_v3_write_bpr0; 1251 break; 1252 case SYS_ICC_DIR_EL1: 1253 if (unlikely(is_read)) 1254 return 0; 1255 fn = __vgic_v3_write_dir; 1256 break; 1257 case SYS_ICC_RPR_EL1: 1258 if (unlikely(!is_read)) 1259 return 0; 1260 fn = __vgic_v3_read_rpr; 1261 break; 1262 case SYS_ICC_CTLR_EL1: 1263 if (is_read) 1264 fn = __vgic_v3_read_ctlr; 1265 else 1266 fn = __vgic_v3_write_ctlr; 1267 break; 1268 case SYS_ICC_PMR_EL1: 1269 if (is_read) 1270 fn = __vgic_v3_read_pmr; 1271 else 1272 fn = __vgic_v3_write_pmr; 1273 break; 1274 default: 1275 return 0; 1276 } 1277 1278 vmcr = __vgic_v3_read_vmcr(); 1279 rt = kvm_vcpu_sys_get_rt(vcpu); 1280 fn(vcpu, vmcr, rt); 1281 1282 __kvm_skip_instr(vcpu); 1283 1284 return 1; 1285 } 1286