1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012-2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <hyp/adjust_pc.h> 8 9 #include <linux/compiler.h> 10 #include <linux/irqchip/arm-gic-v3.h> 11 #include <linux/kvm_host.h> 12 13 #include <asm/kvm_emulate.h> 14 #include <asm/kvm_hyp.h> 15 #include <asm/kvm_mmu.h> 16 17 #define vtr_to_max_lr_idx(v) ((v) & 0xf) 18 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1) 19 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5)) 20 21 u64 __gic_v3_get_lr(unsigned int lr) 22 { 23 switch (lr & 0xf) { 24 case 0: 25 return read_gicreg(ICH_LR0_EL2); 26 case 1: 27 return read_gicreg(ICH_LR1_EL2); 28 case 2: 29 return read_gicreg(ICH_LR2_EL2); 30 case 3: 31 return read_gicreg(ICH_LR3_EL2); 32 case 4: 33 return read_gicreg(ICH_LR4_EL2); 34 case 5: 35 return read_gicreg(ICH_LR5_EL2); 36 case 6: 37 return read_gicreg(ICH_LR6_EL2); 38 case 7: 39 return read_gicreg(ICH_LR7_EL2); 40 case 8: 41 return read_gicreg(ICH_LR8_EL2); 42 case 9: 43 return read_gicreg(ICH_LR9_EL2); 44 case 10: 45 return read_gicreg(ICH_LR10_EL2); 46 case 11: 47 return read_gicreg(ICH_LR11_EL2); 48 case 12: 49 return read_gicreg(ICH_LR12_EL2); 50 case 13: 51 return read_gicreg(ICH_LR13_EL2); 52 case 14: 53 return read_gicreg(ICH_LR14_EL2); 54 case 15: 55 return read_gicreg(ICH_LR15_EL2); 56 } 57 58 unreachable(); 59 } 60 61 static void __gic_v3_set_lr(u64 val, int lr) 62 { 63 switch (lr & 0xf) { 64 case 0: 65 write_gicreg(val, ICH_LR0_EL2); 66 break; 67 case 1: 68 write_gicreg(val, ICH_LR1_EL2); 69 break; 70 case 2: 71 write_gicreg(val, ICH_LR2_EL2); 72 break; 73 case 3: 74 write_gicreg(val, ICH_LR3_EL2); 75 break; 76 case 4: 77 write_gicreg(val, ICH_LR4_EL2); 78 break; 79 case 5: 80 write_gicreg(val, ICH_LR5_EL2); 81 break; 82 case 6: 83 write_gicreg(val, ICH_LR6_EL2); 84 break; 85 case 7: 86 write_gicreg(val, ICH_LR7_EL2); 87 break; 88 case 8: 89 write_gicreg(val, ICH_LR8_EL2); 90 break; 91 case 9: 92 write_gicreg(val, ICH_LR9_EL2); 93 break; 94 case 10: 95 write_gicreg(val, ICH_LR10_EL2); 96 break; 97 case 11: 98 write_gicreg(val, ICH_LR11_EL2); 99 break; 100 case 12: 101 write_gicreg(val, ICH_LR12_EL2); 102 break; 103 case 13: 104 write_gicreg(val, ICH_LR13_EL2); 105 break; 106 case 14: 107 write_gicreg(val, ICH_LR14_EL2); 108 break; 109 case 15: 110 write_gicreg(val, ICH_LR15_EL2); 111 break; 112 } 113 } 114 115 static void __vgic_v3_write_ap0rn(u32 val, int n) 116 { 117 switch (n) { 118 case 0: 119 write_gicreg(val, ICH_AP0R0_EL2); 120 break; 121 case 1: 122 write_gicreg(val, ICH_AP0R1_EL2); 123 break; 124 case 2: 125 write_gicreg(val, ICH_AP0R2_EL2); 126 break; 127 case 3: 128 write_gicreg(val, ICH_AP0R3_EL2); 129 break; 130 } 131 } 132 133 static void __vgic_v3_write_ap1rn(u32 val, int n) 134 { 135 switch (n) { 136 case 0: 137 write_gicreg(val, ICH_AP1R0_EL2); 138 break; 139 case 1: 140 write_gicreg(val, ICH_AP1R1_EL2); 141 break; 142 case 2: 143 write_gicreg(val, ICH_AP1R2_EL2); 144 break; 145 case 3: 146 write_gicreg(val, ICH_AP1R3_EL2); 147 break; 148 } 149 } 150 151 static u32 __vgic_v3_read_ap0rn(int n) 152 { 153 u32 val; 154 155 switch (n) { 156 case 0: 157 val = read_gicreg(ICH_AP0R0_EL2); 158 break; 159 case 1: 160 val = read_gicreg(ICH_AP0R1_EL2); 161 break; 162 case 2: 163 val = read_gicreg(ICH_AP0R2_EL2); 164 break; 165 case 3: 166 val = read_gicreg(ICH_AP0R3_EL2); 167 break; 168 default: 169 unreachable(); 170 } 171 172 return val; 173 } 174 175 static u32 __vgic_v3_read_ap1rn(int n) 176 { 177 u32 val; 178 179 switch (n) { 180 case 0: 181 val = read_gicreg(ICH_AP1R0_EL2); 182 break; 183 case 1: 184 val = read_gicreg(ICH_AP1R1_EL2); 185 break; 186 case 2: 187 val = read_gicreg(ICH_AP1R2_EL2); 188 break; 189 case 3: 190 val = read_gicreg(ICH_AP1R3_EL2); 191 break; 192 default: 193 unreachable(); 194 } 195 196 return val; 197 } 198 199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if) 200 { 201 u64 used_lrs = cpu_if->used_lrs; 202 203 /* 204 * Make sure stores to the GIC via the memory mapped interface 205 * are now visible to the system register interface when reading the 206 * LRs, and when reading back the VMCR on non-VHE systems. 207 */ 208 if (used_lrs || !has_vhe()) { 209 if (!cpu_if->vgic_sre) { 210 dsb(sy); 211 isb(); 212 } 213 } 214 215 if (used_lrs || cpu_if->its_vpe.its_vm) { 216 int i; 217 u32 elrsr; 218 219 elrsr = read_gicreg(ICH_ELRSR_EL2); 220 221 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EL2_En, ICH_HCR_EL2); 222 223 for (i = 0; i < used_lrs; i++) { 224 if (elrsr & (1 << i)) 225 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE; 226 else 227 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i); 228 229 __gic_v3_set_lr(0, i); 230 } 231 } 232 } 233 234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if) 235 { 236 u64 used_lrs = cpu_if->used_lrs; 237 int i; 238 239 if (used_lrs || cpu_if->its_vpe.its_vm) { 240 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); 241 242 for (i = 0; i < used_lrs; i++) 243 __gic_v3_set_lr(cpu_if->vgic_lr[i], i); 244 } 245 246 /* 247 * Ensure that writes to the LRs, and on non-VHE systems ensure that 248 * the write to the VMCR in __vgic_v3_activate_traps(), will have 249 * reached the (re)distributors. This ensure the guest will read the 250 * correct values from the memory-mapped interface. 251 */ 252 if (used_lrs || !has_vhe()) { 253 if (!cpu_if->vgic_sre) { 254 isb(); 255 dsb(sy); 256 } 257 } 258 } 259 260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if) 261 { 262 /* 263 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a 264 * Group0 interrupt (as generated in GICv2 mode) to be 265 * delivered as a FIQ to the guest, with potentially fatal 266 * consequences. So we must make sure that ICC_SRE_EL1 has 267 * been actually programmed with the value we want before 268 * starting to mess with the rest of the GIC, and VMCR_EL2 in 269 * particular. This logic must be called before 270 * __vgic_v3_restore_state(). 271 * 272 * However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is 273 * provisioned at all. In order to prevent illegal accesses to the 274 * system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1 275 * so that the trap bits can take effect. Yes, we *loves* the GIC. 276 */ 277 if (!(cpu_if->vgic_hcr & ICH_HCR_EL2_En)) { 278 write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1); 279 isb(); 280 } else if (!cpu_if->vgic_sre) { 281 write_gicreg(0, ICC_SRE_EL1); 282 isb(); 283 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2); 284 285 286 if (has_vhe()) { 287 /* 288 * Ensure that the write to the VMCR will have reached 289 * the (re)distributors. This ensure the guest will 290 * read the correct values from the memory-mapped 291 * interface. 292 */ 293 isb(); 294 dsb(sy); 295 } 296 } 297 298 /* Only disable SRE if the host implements the GICv2 interface */ 299 if (static_branch_unlikely(&vgic_v3_has_v2_compat)) { 300 /* 301 * Prevent the guest from touching the ICC_SRE_EL1 system 302 * register. Note that this may not have any effect, as 303 * ICC_SRE_EL2.Enable being RAO/WI is a valid implementation. 304 */ 305 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE, 306 ICC_SRE_EL2); 307 } 308 309 /* 310 * If we need to trap system registers, we must write 311 * ICH_HCR_EL2 anyway, even if no interrupts are being 312 * injected. Note that this also applies if we don't expect 313 * any system register access (no vgic at all). 314 */ 315 if (static_branch_unlikely(&vgic_v3_cpuif_trap) || 316 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre) 317 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); 318 } 319 320 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if) 321 { 322 u64 val; 323 324 if (!cpu_if->vgic_sre) { 325 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2); 326 } 327 328 /* Only restore SRE if the host implements the GICv2 interface */ 329 if (static_branch_unlikely(&vgic_v3_has_v2_compat)) { 330 val = read_gicreg(ICC_SRE_EL2); 331 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2); 332 333 if (!cpu_if->vgic_sre) { 334 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */ 335 isb(); 336 write_gicreg(1, ICC_SRE_EL1); 337 } 338 } 339 340 /* 341 * If we were trapping system registers, we enabled the VGIC even if 342 * no interrupts were being injected, and we disable it again here. 343 */ 344 if (static_branch_unlikely(&vgic_v3_cpuif_trap) || 345 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre) 346 write_gicreg(0, ICH_HCR_EL2); 347 } 348 349 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if) 350 { 351 u64 val; 352 u32 nr_pre_bits; 353 354 val = read_gicreg(ICH_VTR_EL2); 355 nr_pre_bits = vtr_to_nr_pre_bits(val); 356 357 switch (nr_pre_bits) { 358 case 7: 359 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3); 360 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2); 361 fallthrough; 362 case 6: 363 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1); 364 fallthrough; 365 default: 366 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0); 367 } 368 369 switch (nr_pre_bits) { 370 case 7: 371 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3); 372 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2); 373 fallthrough; 374 case 6: 375 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1); 376 fallthrough; 377 default: 378 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0); 379 } 380 } 381 382 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) 383 { 384 u64 val; 385 u32 nr_pre_bits; 386 387 val = read_gicreg(ICH_VTR_EL2); 388 nr_pre_bits = vtr_to_nr_pre_bits(val); 389 390 switch (nr_pre_bits) { 391 case 7: 392 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3); 393 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2); 394 fallthrough; 395 case 6: 396 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1); 397 fallthrough; 398 default: 399 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0); 400 } 401 402 switch (nr_pre_bits) { 403 case 7: 404 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3); 405 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2); 406 fallthrough; 407 case 6: 408 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1); 409 fallthrough; 410 default: 411 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0); 412 } 413 } 414 415 void __vgic_v3_init_lrs(void) 416 { 417 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2)); 418 int i; 419 420 for (i = 0; i <= max_lr_idx; i++) 421 __gic_v3_set_lr(0, i); 422 } 423 424 /* 425 * Return the GIC CPU configuration: 426 * - [31:0] ICH_VTR_EL2 427 * - [62:32] RES0 428 * - [63] MMIO (GICv2) capable 429 */ 430 u64 __vgic_v3_get_gic_config(void) 431 { 432 u64 val, sre; 433 unsigned long flags = 0; 434 435 /* 436 * In compat mode, we cannot access ICC_SRE_EL1 at any EL 437 * other than EL1 itself; just return the 438 * ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5 439 * system, so we first check if we have GICv5 support. 440 */ 441 if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) 442 return read_gicreg(ICH_VTR_EL2); 443 444 sre = read_gicreg(ICC_SRE_EL1); 445 /* 446 * To check whether we have a MMIO-based (GICv2 compatible) 447 * CPU interface, we need to disable the system register 448 * view. 449 * 450 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates 451 * that to be able to set ICC_SRE_EL1.SRE to 0, all the 452 * interrupt overrides must be set. You've got to love this. 453 * 454 * As we always run VHE with HCR_xMO set, no extra xMO 455 * manipulation is required in that case. 456 * 457 * To safely disable SRE, we have to prevent any interrupt 458 * from firing (which would be deadly). This only makes sense 459 * on VHE, as interrupts are already masked for nVHE as part 460 * of the exception entry to EL2. 461 */ 462 if (has_vhe()) { 463 flags = local_daif_save(); 464 } else { 465 sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO); 466 isb(); 467 } 468 469 write_gicreg(0, ICC_SRE_EL1); 470 isb(); 471 472 val = read_gicreg(ICC_SRE_EL1); 473 474 write_gicreg(sre, ICC_SRE_EL1); 475 isb(); 476 477 if (has_vhe()) { 478 local_daif_restore(flags); 479 } else { 480 sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0); 481 isb(); 482 } 483 484 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); 485 val |= read_gicreg(ICH_VTR_EL2); 486 487 return val; 488 } 489 490 static void __vgic_v3_compat_mode_enable(void) 491 { 492 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) 493 return; 494 495 sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3); 496 /* Wait for V3 to become enabled */ 497 isb(); 498 } 499 500 static u64 __vgic_v3_read_vmcr(void) 501 { 502 return read_gicreg(ICH_VMCR_EL2); 503 } 504 505 static void __vgic_v3_write_vmcr(u32 vmcr) 506 { 507 write_gicreg(vmcr, ICH_VMCR_EL2); 508 } 509 510 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) 511 { 512 __vgic_v3_save_aprs(cpu_if); 513 if (cpu_if->vgic_sre) 514 cpu_if->vgic_vmcr = __vgic_v3_read_vmcr(); 515 } 516 517 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) 518 { 519 __vgic_v3_compat_mode_enable(); 520 521 /* 522 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen 523 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the 524 * VMCR_EL2 save/restore in the world switch. 525 */ 526 if (cpu_if->vgic_sre) 527 __vgic_v3_write_vmcr(cpu_if->vgic_vmcr); 528 __vgic_v3_restore_aprs(cpu_if); 529 } 530 531 static int __vgic_v3_bpr_min(void) 532 { 533 /* See Pseudocode for VPriorityGroup */ 534 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2)); 535 } 536 537 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu) 538 { 539 u64 esr = kvm_vcpu_get_esr(vcpu); 540 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; 541 542 return crm != 8; 543 } 544 545 #define GICv3_IDLE_PRIORITY 0xff 546 547 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr, 548 u64 *lr_val) 549 { 550 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; 551 u8 priority = GICv3_IDLE_PRIORITY; 552 int i, lr = -1; 553 554 for (i = 0; i < used_lrs; i++) { 555 u64 val = __gic_v3_get_lr(i); 556 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 557 558 /* Not pending in the state? */ 559 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT) 560 continue; 561 562 /* Group-0 interrupt, but Group-0 disabled? */ 563 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK)) 564 continue; 565 566 /* Group-1 interrupt, but Group-1 disabled? */ 567 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK)) 568 continue; 569 570 /* Not the highest priority? */ 571 if (lr_prio >= priority) 572 continue; 573 574 /* This is a candidate */ 575 priority = lr_prio; 576 *lr_val = val; 577 lr = i; 578 } 579 580 if (lr == -1) 581 *lr_val = ICC_IAR1_EL1_SPURIOUS; 582 583 return lr; 584 } 585 586 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid, 587 u64 *lr_val) 588 { 589 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; 590 int i; 591 592 for (i = 0; i < used_lrs; i++) { 593 u64 val = __gic_v3_get_lr(i); 594 595 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid && 596 (val & ICH_LR_ACTIVE_BIT)) { 597 *lr_val = val; 598 return i; 599 } 600 } 601 602 *lr_val = ICC_IAR1_EL1_SPURIOUS; 603 return -1; 604 } 605 606 static int __vgic_v3_get_highest_active_priority(void) 607 { 608 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); 609 u32 hap = 0; 610 int i; 611 612 for (i = 0; i < nr_apr_regs; i++) { 613 u32 val; 614 615 /* 616 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers 617 * contain the active priority levels for this VCPU 618 * for the maximum number of supported priority 619 * levels, and we return the full priority level only 620 * if the BPR is programmed to its minimum, otherwise 621 * we return a combination of the priority level and 622 * subpriority, as determined by the setting of the 623 * BPR, but without the full subpriority. 624 */ 625 val = __vgic_v3_read_ap0rn(i); 626 val |= __vgic_v3_read_ap1rn(i); 627 if (!val) { 628 hap += 32; 629 continue; 630 } 631 632 return (hap + __ffs(val)) << __vgic_v3_bpr_min(); 633 } 634 635 return GICv3_IDLE_PRIORITY; 636 } 637 638 static unsigned int __vgic_v3_get_bpr0(u32 vmcr) 639 { 640 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT; 641 } 642 643 static unsigned int __vgic_v3_get_bpr1(u32 vmcr) 644 { 645 unsigned int bpr; 646 647 if (vmcr & ICH_VMCR_CBPR_MASK) { 648 bpr = __vgic_v3_get_bpr0(vmcr); 649 if (bpr < 7) 650 bpr++; 651 } else { 652 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT; 653 } 654 655 return bpr; 656 } 657 658 /* 659 * Convert a priority to a preemption level, taking the relevant BPR 660 * into account by zeroing the sub-priority bits. 661 */ 662 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp) 663 { 664 unsigned int bpr; 665 666 if (!grp) 667 bpr = __vgic_v3_get_bpr0(vmcr) + 1; 668 else 669 bpr = __vgic_v3_get_bpr1(vmcr); 670 671 return pri & (GENMASK(7, 0) << bpr); 672 } 673 674 /* 675 * The priority value is independent of any of the BPR values, so we 676 * normalize it using the minimal BPR value. This guarantees that no 677 * matter what the guest does with its BPR, we can always set/get the 678 * same value of a priority. 679 */ 680 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp) 681 { 682 u8 pre, ap; 683 u32 val; 684 int apr; 685 686 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp); 687 ap = pre >> __vgic_v3_bpr_min(); 688 apr = ap / 32; 689 690 if (!grp) { 691 val = __vgic_v3_read_ap0rn(apr); 692 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr); 693 } else { 694 val = __vgic_v3_read_ap1rn(apr); 695 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr); 696 } 697 } 698 699 static int __vgic_v3_clear_highest_active_priority(void) 700 { 701 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); 702 u32 hap = 0; 703 int i; 704 705 for (i = 0; i < nr_apr_regs; i++) { 706 u32 ap0, ap1; 707 int c0, c1; 708 709 ap0 = __vgic_v3_read_ap0rn(i); 710 ap1 = __vgic_v3_read_ap1rn(i); 711 if (!ap0 && !ap1) { 712 hap += 32; 713 continue; 714 } 715 716 c0 = ap0 ? __ffs(ap0) : 32; 717 c1 = ap1 ? __ffs(ap1) : 32; 718 719 /* Always clear the LSB, which is the highest priority */ 720 if (c0 < c1) { 721 ap0 &= ~BIT(c0); 722 __vgic_v3_write_ap0rn(ap0, i); 723 hap += c0; 724 } else { 725 ap1 &= ~BIT(c1); 726 __vgic_v3_write_ap1rn(ap1, i); 727 hap += c1; 728 } 729 730 /* Rescale to 8 bits of priority */ 731 return hap << __vgic_v3_bpr_min(); 732 } 733 734 return GICv3_IDLE_PRIORITY; 735 } 736 737 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 738 { 739 u64 lr_val; 740 u8 lr_prio, pmr; 741 int lr, grp; 742 743 grp = __vgic_v3_get_group(vcpu); 744 745 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); 746 if (lr < 0) 747 goto spurious; 748 749 if (grp != !!(lr_val & ICH_LR_GROUP)) 750 goto spurious; 751 752 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT; 753 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 754 if (pmr <= lr_prio) 755 goto spurious; 756 757 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp)) 758 goto spurious; 759 760 lr_val &= ~ICH_LR_STATE; 761 lr_val |= ICH_LR_ACTIVE_BIT; 762 __gic_v3_set_lr(lr_val, lr); 763 __vgic_v3_set_active_priority(lr_prio, vmcr, grp); 764 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); 765 return; 766 767 spurious: 768 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS); 769 } 770 771 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val) 772 { 773 lr_val &= ~ICH_LR_ACTIVE_BIT; 774 if (lr_val & ICH_LR_HW) { 775 u32 pid; 776 777 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT; 778 gic_write_dir(pid); 779 } 780 781 __gic_v3_set_lr(lr_val, lr); 782 } 783 784 static void __vgic_v3_bump_eoicount(void) 785 { 786 u32 hcr; 787 788 hcr = read_gicreg(ICH_HCR_EL2); 789 hcr += 1 << ICH_HCR_EL2_EOIcount_SHIFT; 790 write_gicreg(hcr, ICH_HCR_EL2); 791 } 792 793 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 794 { 795 u32 vid = vcpu_get_reg(vcpu, rt); 796 u64 lr_val; 797 int lr; 798 799 /* EOImode == 0, nothing to be done here */ 800 if (!(vmcr & ICH_VMCR_EOIM_MASK)) 801 return; 802 803 /* No deactivate to be performed on an LPI */ 804 if (vid >= VGIC_MIN_LPI) 805 return; 806 807 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); 808 if (lr == -1) { 809 __vgic_v3_bump_eoicount(); 810 return; 811 } 812 813 __vgic_v3_clear_active_lr(lr, lr_val); 814 } 815 816 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 817 { 818 u32 vid = vcpu_get_reg(vcpu, rt); 819 u64 lr_val; 820 u8 lr_prio, act_prio; 821 int lr, grp; 822 823 grp = __vgic_v3_get_group(vcpu); 824 825 /* Drop priority in any case */ 826 act_prio = __vgic_v3_clear_highest_active_priority(); 827 828 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); 829 if (lr == -1) { 830 /* Do not bump EOIcount for LPIs that aren't in the LRs */ 831 if (!(vid >= VGIC_MIN_LPI)) 832 __vgic_v3_bump_eoicount(); 833 return; 834 } 835 836 /* EOImode == 1 and not an LPI, nothing to be done here */ 837 if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI)) 838 return; 839 840 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 841 842 /* If priorities or group do not match, the guest has fscked-up. */ 843 if (grp != !!(lr_val & ICH_LR_GROUP) || 844 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio) 845 return; 846 847 /* Let's now perform the deactivation */ 848 __vgic_v3_clear_active_lr(lr, lr_val); 849 } 850 851 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 852 { 853 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK)); 854 } 855 856 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 857 { 858 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK)); 859 } 860 861 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 862 { 863 u64 val = vcpu_get_reg(vcpu, rt); 864 865 if (val & 1) 866 vmcr |= ICH_VMCR_ENG0_MASK; 867 else 868 vmcr &= ~ICH_VMCR_ENG0_MASK; 869 870 __vgic_v3_write_vmcr(vmcr); 871 } 872 873 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 874 { 875 u64 val = vcpu_get_reg(vcpu, rt); 876 877 if (val & 1) 878 vmcr |= ICH_VMCR_ENG1_MASK; 879 else 880 vmcr &= ~ICH_VMCR_ENG1_MASK; 881 882 __vgic_v3_write_vmcr(vmcr); 883 } 884 885 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 886 { 887 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr)); 888 } 889 890 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 891 { 892 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr)); 893 } 894 895 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 896 { 897 u64 val = vcpu_get_reg(vcpu, rt); 898 u8 bpr_min = __vgic_v3_bpr_min() - 1; 899 900 /* Enforce BPR limiting */ 901 if (val < bpr_min) 902 val = bpr_min; 903 904 val <<= ICH_VMCR_BPR0_SHIFT; 905 val &= ICH_VMCR_BPR0_MASK; 906 vmcr &= ~ICH_VMCR_BPR0_MASK; 907 vmcr |= val; 908 909 __vgic_v3_write_vmcr(vmcr); 910 } 911 912 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 913 { 914 u64 val = vcpu_get_reg(vcpu, rt); 915 u8 bpr_min = __vgic_v3_bpr_min(); 916 917 if (vmcr & ICH_VMCR_CBPR_MASK) 918 return; 919 920 /* Enforce BPR limiting */ 921 if (val < bpr_min) 922 val = bpr_min; 923 924 val <<= ICH_VMCR_BPR1_SHIFT; 925 val &= ICH_VMCR_BPR1_MASK; 926 vmcr &= ~ICH_VMCR_BPR1_MASK; 927 vmcr |= val; 928 929 __vgic_v3_write_vmcr(vmcr); 930 } 931 932 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n) 933 { 934 u32 val; 935 936 if (!__vgic_v3_get_group(vcpu)) 937 val = __vgic_v3_read_ap0rn(n); 938 else 939 val = __vgic_v3_read_ap1rn(n); 940 941 vcpu_set_reg(vcpu, rt, val); 942 } 943 944 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n) 945 { 946 u32 val = vcpu_get_reg(vcpu, rt); 947 948 if (!__vgic_v3_get_group(vcpu)) 949 __vgic_v3_write_ap0rn(val, n); 950 else 951 __vgic_v3_write_ap1rn(val, n); 952 } 953 954 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu, 955 u32 vmcr, int rt) 956 { 957 __vgic_v3_read_apxrn(vcpu, rt, 0); 958 } 959 960 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu, 961 u32 vmcr, int rt) 962 { 963 __vgic_v3_read_apxrn(vcpu, rt, 1); 964 } 965 966 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 967 { 968 __vgic_v3_read_apxrn(vcpu, rt, 2); 969 } 970 971 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 972 { 973 __vgic_v3_read_apxrn(vcpu, rt, 3); 974 } 975 976 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 977 { 978 __vgic_v3_write_apxrn(vcpu, rt, 0); 979 } 980 981 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 982 { 983 __vgic_v3_write_apxrn(vcpu, rt, 1); 984 } 985 986 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 987 { 988 __vgic_v3_write_apxrn(vcpu, rt, 2); 989 } 990 991 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 992 { 993 __vgic_v3_write_apxrn(vcpu, rt, 3); 994 } 995 996 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 997 { 998 u64 lr_val; 999 int lr, lr_grp, grp; 1000 1001 grp = __vgic_v3_get_group(vcpu); 1002 1003 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); 1004 if (lr == -1) 1005 goto spurious; 1006 1007 lr_grp = !!(lr_val & ICH_LR_GROUP); 1008 if (lr_grp != grp) 1009 lr_val = ICC_IAR1_EL1_SPURIOUS; 1010 1011 spurious: 1012 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); 1013 } 1014 1015 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1016 { 1017 vmcr &= ICH_VMCR_PMR_MASK; 1018 vmcr >>= ICH_VMCR_PMR_SHIFT; 1019 vcpu_set_reg(vcpu, rt, vmcr); 1020 } 1021 1022 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1023 { 1024 u32 val = vcpu_get_reg(vcpu, rt); 1025 1026 val <<= ICH_VMCR_PMR_SHIFT; 1027 val &= ICH_VMCR_PMR_MASK; 1028 vmcr &= ~ICH_VMCR_PMR_MASK; 1029 vmcr |= val; 1030 1031 write_gicreg(vmcr, ICH_VMCR_EL2); 1032 } 1033 1034 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1035 { 1036 u32 val = __vgic_v3_get_highest_active_priority(); 1037 vcpu_set_reg(vcpu, rt, val); 1038 } 1039 1040 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1041 { 1042 u32 vtr, val; 1043 1044 vtr = read_gicreg(ICH_VTR_EL2); 1045 /* PRIbits */ 1046 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT; 1047 /* IDbits */ 1048 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT; 1049 /* A3V */ 1050 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT; 1051 /* EOImode */ 1052 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT; 1053 /* CBPR */ 1054 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT; 1055 1056 vcpu_set_reg(vcpu, rt, val); 1057 } 1058 1059 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1060 { 1061 u32 val = vcpu_get_reg(vcpu, rt); 1062 1063 if (val & ICC_CTLR_EL1_CBPR_MASK) 1064 vmcr |= ICH_VMCR_CBPR_MASK; 1065 else 1066 vmcr &= ~ICH_VMCR_CBPR_MASK; 1067 1068 if (val & ICC_CTLR_EL1_EOImode_MASK) 1069 vmcr |= ICH_VMCR_EOIM_MASK; 1070 else 1071 vmcr &= ~ICH_VMCR_EOIM_MASK; 1072 1073 write_gicreg(vmcr, ICH_VMCR_EL2); 1074 } 1075 1076 static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu, 1077 u32 sysreg, bool is_read) 1078 { 1079 u64 ich_hcr; 1080 1081 if (!is_nested_ctxt(vcpu)) 1082 return false; 1083 1084 ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2); 1085 1086 switch (sysreg) { 1087 case SYS_ICC_IGRPEN0_EL1: 1088 if (is_read && 1089 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1)) 1090 return true; 1091 1092 if (!is_read && 1093 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1)) 1094 return true; 1095 1096 fallthrough; 1097 1098 case SYS_ICC_AP0Rn_EL1(0): 1099 case SYS_ICC_AP0Rn_EL1(1): 1100 case SYS_ICC_AP0Rn_EL1(2): 1101 case SYS_ICC_AP0Rn_EL1(3): 1102 case SYS_ICC_BPR0_EL1: 1103 case SYS_ICC_EOIR0_EL1: 1104 case SYS_ICC_HPPIR0_EL1: 1105 case SYS_ICC_IAR0_EL1: 1106 return ich_hcr & ICH_HCR_EL2_TALL0; 1107 1108 case SYS_ICC_IGRPEN1_EL1: 1109 if (is_read && 1110 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1)) 1111 return true; 1112 1113 if (!is_read && 1114 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1)) 1115 return true; 1116 1117 fallthrough; 1118 1119 case SYS_ICC_AP1Rn_EL1(0): 1120 case SYS_ICC_AP1Rn_EL1(1): 1121 case SYS_ICC_AP1Rn_EL1(2): 1122 case SYS_ICC_AP1Rn_EL1(3): 1123 case SYS_ICC_BPR1_EL1: 1124 case SYS_ICC_EOIR1_EL1: 1125 case SYS_ICC_HPPIR1_EL1: 1126 case SYS_ICC_IAR1_EL1: 1127 return ich_hcr & ICH_HCR_EL2_TALL1; 1128 1129 case SYS_ICC_DIR_EL1: 1130 if (ich_hcr & ICH_HCR_EL2_TDIR) 1131 return true; 1132 1133 fallthrough; 1134 1135 case SYS_ICC_RPR_EL1: 1136 case SYS_ICC_CTLR_EL1: 1137 case SYS_ICC_PMR_EL1: 1138 return ich_hcr & ICH_HCR_EL2_TC; 1139 1140 default: 1141 return false; 1142 } 1143 } 1144 1145 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu) 1146 { 1147 int rt; 1148 u64 esr; 1149 u32 vmcr; 1150 void (*fn)(struct kvm_vcpu *, u32, int); 1151 bool is_read; 1152 u32 sysreg; 1153 1154 if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) 1155 return 0; 1156 1157 esr = kvm_vcpu_get_esr(vcpu); 1158 if (vcpu_mode_is_32bit(vcpu)) { 1159 if (!kvm_condition_valid(vcpu)) { 1160 __kvm_skip_instr(vcpu); 1161 return 1; 1162 } 1163 1164 sysreg = esr_cp15_to_sysreg(esr); 1165 } else { 1166 sysreg = esr_sys64_to_sysreg(esr); 1167 } 1168 1169 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ; 1170 1171 if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read)) 1172 return 0; 1173 1174 switch (sysreg) { 1175 case SYS_ICC_IAR0_EL1: 1176 case SYS_ICC_IAR1_EL1: 1177 if (unlikely(!is_read)) 1178 return 0; 1179 fn = __vgic_v3_read_iar; 1180 break; 1181 case SYS_ICC_EOIR0_EL1: 1182 case SYS_ICC_EOIR1_EL1: 1183 if (unlikely(is_read)) 1184 return 0; 1185 fn = __vgic_v3_write_eoir; 1186 break; 1187 case SYS_ICC_IGRPEN1_EL1: 1188 if (is_read) 1189 fn = __vgic_v3_read_igrpen1; 1190 else 1191 fn = __vgic_v3_write_igrpen1; 1192 break; 1193 case SYS_ICC_BPR1_EL1: 1194 if (is_read) 1195 fn = __vgic_v3_read_bpr1; 1196 else 1197 fn = __vgic_v3_write_bpr1; 1198 break; 1199 case SYS_ICC_AP0Rn_EL1(0): 1200 case SYS_ICC_AP1Rn_EL1(0): 1201 if (is_read) 1202 fn = __vgic_v3_read_apxr0; 1203 else 1204 fn = __vgic_v3_write_apxr0; 1205 break; 1206 case SYS_ICC_AP0Rn_EL1(1): 1207 case SYS_ICC_AP1Rn_EL1(1): 1208 if (is_read) 1209 fn = __vgic_v3_read_apxr1; 1210 else 1211 fn = __vgic_v3_write_apxr1; 1212 break; 1213 case SYS_ICC_AP0Rn_EL1(2): 1214 case SYS_ICC_AP1Rn_EL1(2): 1215 if (is_read) 1216 fn = __vgic_v3_read_apxr2; 1217 else 1218 fn = __vgic_v3_write_apxr2; 1219 break; 1220 case SYS_ICC_AP0Rn_EL1(3): 1221 case SYS_ICC_AP1Rn_EL1(3): 1222 if (is_read) 1223 fn = __vgic_v3_read_apxr3; 1224 else 1225 fn = __vgic_v3_write_apxr3; 1226 break; 1227 case SYS_ICC_HPPIR0_EL1: 1228 case SYS_ICC_HPPIR1_EL1: 1229 if (unlikely(!is_read)) 1230 return 0; 1231 fn = __vgic_v3_read_hppir; 1232 break; 1233 case SYS_ICC_IGRPEN0_EL1: 1234 if (is_read) 1235 fn = __vgic_v3_read_igrpen0; 1236 else 1237 fn = __vgic_v3_write_igrpen0; 1238 break; 1239 case SYS_ICC_BPR0_EL1: 1240 if (is_read) 1241 fn = __vgic_v3_read_bpr0; 1242 else 1243 fn = __vgic_v3_write_bpr0; 1244 break; 1245 case SYS_ICC_DIR_EL1: 1246 if (unlikely(is_read)) 1247 return 0; 1248 fn = __vgic_v3_write_dir; 1249 break; 1250 case SYS_ICC_RPR_EL1: 1251 if (unlikely(!is_read)) 1252 return 0; 1253 fn = __vgic_v3_read_rpr; 1254 break; 1255 case SYS_ICC_CTLR_EL1: 1256 if (is_read) 1257 fn = __vgic_v3_read_ctlr; 1258 else 1259 fn = __vgic_v3_write_ctlr; 1260 break; 1261 case SYS_ICC_PMR_EL1: 1262 if (is_read) 1263 fn = __vgic_v3_read_pmr; 1264 else 1265 fn = __vgic_v3_write_pmr; 1266 break; 1267 default: 1268 return 0; 1269 } 1270 1271 vmcr = __vgic_v3_read_vmcr(); 1272 rt = kvm_vcpu_sys_get_rt(vcpu); 1273 fn(vcpu, vmcr, rt); 1274 1275 __kvm_skip_instr(vcpu); 1276 1277 return 1; 1278 } 1279