1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012-2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <hyp/adjust_pc.h> 8 9 #include <linux/compiler.h> 10 #include <linux/irqchip/arm-gic-v3.h> 11 #include <linux/kvm_host.h> 12 13 #include <asm/kvm_emulate.h> 14 #include <asm/kvm_hyp.h> 15 #include <asm/kvm_mmu.h> 16 17 #define vtr_to_max_lr_idx(v) ((v) & 0xf) 18 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1) 19 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5)) 20 21 u64 __gic_v3_get_lr(unsigned int lr) 22 { 23 switch (lr & 0xf) { 24 case 0: 25 return read_gicreg(ICH_LR0_EL2); 26 case 1: 27 return read_gicreg(ICH_LR1_EL2); 28 case 2: 29 return read_gicreg(ICH_LR2_EL2); 30 case 3: 31 return read_gicreg(ICH_LR3_EL2); 32 case 4: 33 return read_gicreg(ICH_LR4_EL2); 34 case 5: 35 return read_gicreg(ICH_LR5_EL2); 36 case 6: 37 return read_gicreg(ICH_LR6_EL2); 38 case 7: 39 return read_gicreg(ICH_LR7_EL2); 40 case 8: 41 return read_gicreg(ICH_LR8_EL2); 42 case 9: 43 return read_gicreg(ICH_LR9_EL2); 44 case 10: 45 return read_gicreg(ICH_LR10_EL2); 46 case 11: 47 return read_gicreg(ICH_LR11_EL2); 48 case 12: 49 return read_gicreg(ICH_LR12_EL2); 50 case 13: 51 return read_gicreg(ICH_LR13_EL2); 52 case 14: 53 return read_gicreg(ICH_LR14_EL2); 54 case 15: 55 return read_gicreg(ICH_LR15_EL2); 56 } 57 58 unreachable(); 59 } 60 61 static void __gic_v3_set_lr(u64 val, int lr) 62 { 63 switch (lr & 0xf) { 64 case 0: 65 write_gicreg(val, ICH_LR0_EL2); 66 break; 67 case 1: 68 write_gicreg(val, ICH_LR1_EL2); 69 break; 70 case 2: 71 write_gicreg(val, ICH_LR2_EL2); 72 break; 73 case 3: 74 write_gicreg(val, ICH_LR3_EL2); 75 break; 76 case 4: 77 write_gicreg(val, ICH_LR4_EL2); 78 break; 79 case 5: 80 write_gicreg(val, ICH_LR5_EL2); 81 break; 82 case 6: 83 write_gicreg(val, ICH_LR6_EL2); 84 break; 85 case 7: 86 write_gicreg(val, ICH_LR7_EL2); 87 break; 88 case 8: 89 write_gicreg(val, ICH_LR8_EL2); 90 break; 91 case 9: 92 write_gicreg(val, ICH_LR9_EL2); 93 break; 94 case 10: 95 write_gicreg(val, ICH_LR10_EL2); 96 break; 97 case 11: 98 write_gicreg(val, ICH_LR11_EL2); 99 break; 100 case 12: 101 write_gicreg(val, ICH_LR12_EL2); 102 break; 103 case 13: 104 write_gicreg(val, ICH_LR13_EL2); 105 break; 106 case 14: 107 write_gicreg(val, ICH_LR14_EL2); 108 break; 109 case 15: 110 write_gicreg(val, ICH_LR15_EL2); 111 break; 112 } 113 } 114 115 static void __vgic_v3_write_ap0rn(u32 val, int n) 116 { 117 switch (n) { 118 case 0: 119 write_gicreg(val, ICH_AP0R0_EL2); 120 break; 121 case 1: 122 write_gicreg(val, ICH_AP0R1_EL2); 123 break; 124 case 2: 125 write_gicreg(val, ICH_AP0R2_EL2); 126 break; 127 case 3: 128 write_gicreg(val, ICH_AP0R3_EL2); 129 break; 130 } 131 } 132 133 static void __vgic_v3_write_ap1rn(u32 val, int n) 134 { 135 switch (n) { 136 case 0: 137 write_gicreg(val, ICH_AP1R0_EL2); 138 break; 139 case 1: 140 write_gicreg(val, ICH_AP1R1_EL2); 141 break; 142 case 2: 143 write_gicreg(val, ICH_AP1R2_EL2); 144 break; 145 case 3: 146 write_gicreg(val, ICH_AP1R3_EL2); 147 break; 148 } 149 } 150 151 static u32 __vgic_v3_read_ap0rn(int n) 152 { 153 u32 val; 154 155 switch (n) { 156 case 0: 157 val = read_gicreg(ICH_AP0R0_EL2); 158 break; 159 case 1: 160 val = read_gicreg(ICH_AP0R1_EL2); 161 break; 162 case 2: 163 val = read_gicreg(ICH_AP0R2_EL2); 164 break; 165 case 3: 166 val = read_gicreg(ICH_AP0R3_EL2); 167 break; 168 default: 169 unreachable(); 170 } 171 172 return val; 173 } 174 175 static u32 __vgic_v3_read_ap1rn(int n) 176 { 177 u32 val; 178 179 switch (n) { 180 case 0: 181 val = read_gicreg(ICH_AP1R0_EL2); 182 break; 183 case 1: 184 val = read_gicreg(ICH_AP1R1_EL2); 185 break; 186 case 2: 187 val = read_gicreg(ICH_AP1R2_EL2); 188 break; 189 case 3: 190 val = read_gicreg(ICH_AP1R3_EL2); 191 break; 192 default: 193 unreachable(); 194 } 195 196 return val; 197 } 198 199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if) 200 { 201 u64 used_lrs = cpu_if->used_lrs; 202 203 /* 204 * Make sure stores to the GIC via the memory mapped interface 205 * are now visible to the system register interface when reading the 206 * LRs, and when reading back the VMCR on non-VHE systems. 207 */ 208 if (used_lrs || !has_vhe()) { 209 if (!cpu_if->vgic_sre) { 210 dsb(sy); 211 isb(); 212 } 213 } 214 215 if (used_lrs || cpu_if->its_vpe.its_vm) { 216 int i; 217 u32 elrsr; 218 219 elrsr = read_gicreg(ICH_ELRSR_EL2); 220 221 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EL2_En, ICH_HCR_EL2); 222 223 for (i = 0; i < used_lrs; i++) { 224 if (elrsr & (1 << i)) 225 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE; 226 else 227 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i); 228 229 __gic_v3_set_lr(0, i); 230 } 231 } 232 } 233 234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if) 235 { 236 u64 used_lrs = cpu_if->used_lrs; 237 int i; 238 239 if (used_lrs || cpu_if->its_vpe.its_vm) { 240 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); 241 242 for (i = 0; i < used_lrs; i++) 243 __gic_v3_set_lr(cpu_if->vgic_lr[i], i); 244 } 245 246 /* 247 * Ensure that writes to the LRs, and on non-VHE systems ensure that 248 * the write to the VMCR in __vgic_v3_activate_traps(), will have 249 * reached the (re)distributors. This ensure the guest will read the 250 * correct values from the memory-mapped interface. 251 */ 252 if (used_lrs || !has_vhe()) { 253 if (!cpu_if->vgic_sre) { 254 isb(); 255 dsb(sy); 256 } 257 } 258 } 259 260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if) 261 { 262 /* 263 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a 264 * Group0 interrupt (as generated in GICv2 mode) to be 265 * delivered as a FIQ to the guest, with potentially fatal 266 * consequences. So we must make sure that ICC_SRE_EL1 has 267 * been actually programmed with the value we want before 268 * starting to mess with the rest of the GIC, and VMCR_EL2 in 269 * particular. This logic must be called before 270 * __vgic_v3_restore_state(). 271 * 272 * However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is 273 * provisioned at all. In order to prevent illegal accesses to the 274 * system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1 275 * so that the trap bits can take effect. Yes, we *loves* the GIC. 276 */ 277 if (!(cpu_if->vgic_hcr & ICH_HCR_EL2_En)) { 278 write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1); 279 isb(); 280 } else if (!cpu_if->vgic_sre) { 281 write_gicreg(0, ICC_SRE_EL1); 282 isb(); 283 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2); 284 285 286 if (has_vhe()) { 287 /* 288 * Ensure that the write to the VMCR will have reached 289 * the (re)distributors. This ensure the guest will 290 * read the correct values from the memory-mapped 291 * interface. 292 */ 293 isb(); 294 dsb(sy); 295 } 296 } 297 298 /* 299 * Prevent the guest from touching the ICC_SRE_EL1 system 300 * register. Note that this may not have any effect, as 301 * ICC_SRE_EL2.Enable being RAO/WI is a valid implementation. 302 */ 303 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE, 304 ICC_SRE_EL2); 305 306 /* 307 * If we need to trap system registers, we must write 308 * ICH_HCR_EL2 anyway, even if no interrupts are being 309 * injected. Note that this also applies if we don't expect 310 * any system register access (no vgic at all). 311 */ 312 if (static_branch_unlikely(&vgic_v3_cpuif_trap) || 313 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre) 314 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); 315 } 316 317 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if) 318 { 319 u64 val; 320 321 if (!cpu_if->vgic_sre) { 322 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2); 323 } 324 325 val = read_gicreg(ICC_SRE_EL2); 326 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2); 327 328 if (!cpu_if->vgic_sre) { 329 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */ 330 isb(); 331 write_gicreg(1, ICC_SRE_EL1); 332 } 333 334 /* 335 * If we were trapping system registers, we enabled the VGIC even if 336 * no interrupts were being injected, and we disable it again here. 337 */ 338 if (static_branch_unlikely(&vgic_v3_cpuif_trap) || 339 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre) 340 write_gicreg(0, ICH_HCR_EL2); 341 } 342 343 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if) 344 { 345 u64 val; 346 u32 nr_pre_bits; 347 348 val = read_gicreg(ICH_VTR_EL2); 349 nr_pre_bits = vtr_to_nr_pre_bits(val); 350 351 switch (nr_pre_bits) { 352 case 7: 353 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3); 354 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2); 355 fallthrough; 356 case 6: 357 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1); 358 fallthrough; 359 default: 360 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0); 361 } 362 363 switch (nr_pre_bits) { 364 case 7: 365 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3); 366 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2); 367 fallthrough; 368 case 6: 369 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1); 370 fallthrough; 371 default: 372 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0); 373 } 374 } 375 376 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) 377 { 378 u64 val; 379 u32 nr_pre_bits; 380 381 val = read_gicreg(ICH_VTR_EL2); 382 nr_pre_bits = vtr_to_nr_pre_bits(val); 383 384 switch (nr_pre_bits) { 385 case 7: 386 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3); 387 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2); 388 fallthrough; 389 case 6: 390 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1); 391 fallthrough; 392 default: 393 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0); 394 } 395 396 switch (nr_pre_bits) { 397 case 7: 398 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3); 399 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2); 400 fallthrough; 401 case 6: 402 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1); 403 fallthrough; 404 default: 405 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0); 406 } 407 } 408 409 void __vgic_v3_init_lrs(void) 410 { 411 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2)); 412 int i; 413 414 for (i = 0; i <= max_lr_idx; i++) 415 __gic_v3_set_lr(0, i); 416 } 417 418 /* 419 * Return the GIC CPU configuration: 420 * - [31:0] ICH_VTR_EL2 421 * - [62:32] RES0 422 * - [63] MMIO (GICv2) capable 423 */ 424 u64 __vgic_v3_get_gic_config(void) 425 { 426 u64 val, sre = read_gicreg(ICC_SRE_EL1); 427 unsigned long flags = 0; 428 429 /* 430 * To check whether we have a MMIO-based (GICv2 compatible) 431 * CPU interface, we need to disable the system register 432 * view. 433 * 434 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates 435 * that to be able to set ICC_SRE_EL1.SRE to 0, all the 436 * interrupt overrides must be set. You've got to love this. 437 * 438 * As we always run VHE with HCR_xMO set, no extra xMO 439 * manipulation is required in that case. 440 * 441 * To safely disable SRE, we have to prevent any interrupt 442 * from firing (which would be deadly). This only makes sense 443 * on VHE, as interrupts are already masked for nVHE as part 444 * of the exception entry to EL2. 445 */ 446 if (has_vhe()) { 447 flags = local_daif_save(); 448 } else { 449 sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO); 450 isb(); 451 } 452 453 write_gicreg(0, ICC_SRE_EL1); 454 isb(); 455 456 val = read_gicreg(ICC_SRE_EL1); 457 458 write_gicreg(sre, ICC_SRE_EL1); 459 isb(); 460 461 if (has_vhe()) { 462 local_daif_restore(flags); 463 } else { 464 sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0); 465 isb(); 466 } 467 468 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); 469 val |= read_gicreg(ICH_VTR_EL2); 470 471 return val; 472 } 473 474 static u64 __vgic_v3_read_vmcr(void) 475 { 476 return read_gicreg(ICH_VMCR_EL2); 477 } 478 479 static void __vgic_v3_write_vmcr(u32 vmcr) 480 { 481 write_gicreg(vmcr, ICH_VMCR_EL2); 482 } 483 484 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) 485 { 486 __vgic_v3_save_aprs(cpu_if); 487 if (cpu_if->vgic_sre) 488 cpu_if->vgic_vmcr = __vgic_v3_read_vmcr(); 489 } 490 491 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) 492 { 493 /* 494 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen 495 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the 496 * VMCR_EL2 save/restore in the world switch. 497 */ 498 if (cpu_if->vgic_sre) 499 __vgic_v3_write_vmcr(cpu_if->vgic_vmcr); 500 __vgic_v3_restore_aprs(cpu_if); 501 } 502 503 static int __vgic_v3_bpr_min(void) 504 { 505 /* See Pseudocode for VPriorityGroup */ 506 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2)); 507 } 508 509 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu) 510 { 511 u64 esr = kvm_vcpu_get_esr(vcpu); 512 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; 513 514 return crm != 8; 515 } 516 517 #define GICv3_IDLE_PRIORITY 0xff 518 519 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr, 520 u64 *lr_val) 521 { 522 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; 523 u8 priority = GICv3_IDLE_PRIORITY; 524 int i, lr = -1; 525 526 for (i = 0; i < used_lrs; i++) { 527 u64 val = __gic_v3_get_lr(i); 528 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 529 530 /* Not pending in the state? */ 531 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT) 532 continue; 533 534 /* Group-0 interrupt, but Group-0 disabled? */ 535 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK)) 536 continue; 537 538 /* Group-1 interrupt, but Group-1 disabled? */ 539 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK)) 540 continue; 541 542 /* Not the highest priority? */ 543 if (lr_prio >= priority) 544 continue; 545 546 /* This is a candidate */ 547 priority = lr_prio; 548 *lr_val = val; 549 lr = i; 550 } 551 552 if (lr == -1) 553 *lr_val = ICC_IAR1_EL1_SPURIOUS; 554 555 return lr; 556 } 557 558 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid, 559 u64 *lr_val) 560 { 561 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; 562 int i; 563 564 for (i = 0; i < used_lrs; i++) { 565 u64 val = __gic_v3_get_lr(i); 566 567 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid && 568 (val & ICH_LR_ACTIVE_BIT)) { 569 *lr_val = val; 570 return i; 571 } 572 } 573 574 *lr_val = ICC_IAR1_EL1_SPURIOUS; 575 return -1; 576 } 577 578 static int __vgic_v3_get_highest_active_priority(void) 579 { 580 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); 581 u32 hap = 0; 582 int i; 583 584 for (i = 0; i < nr_apr_regs; i++) { 585 u32 val; 586 587 /* 588 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers 589 * contain the active priority levels for this VCPU 590 * for the maximum number of supported priority 591 * levels, and we return the full priority level only 592 * if the BPR is programmed to its minimum, otherwise 593 * we return a combination of the priority level and 594 * subpriority, as determined by the setting of the 595 * BPR, but without the full subpriority. 596 */ 597 val = __vgic_v3_read_ap0rn(i); 598 val |= __vgic_v3_read_ap1rn(i); 599 if (!val) { 600 hap += 32; 601 continue; 602 } 603 604 return (hap + __ffs(val)) << __vgic_v3_bpr_min(); 605 } 606 607 return GICv3_IDLE_PRIORITY; 608 } 609 610 static unsigned int __vgic_v3_get_bpr0(u32 vmcr) 611 { 612 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT; 613 } 614 615 static unsigned int __vgic_v3_get_bpr1(u32 vmcr) 616 { 617 unsigned int bpr; 618 619 if (vmcr & ICH_VMCR_CBPR_MASK) { 620 bpr = __vgic_v3_get_bpr0(vmcr); 621 if (bpr < 7) 622 bpr++; 623 } else { 624 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT; 625 } 626 627 return bpr; 628 } 629 630 /* 631 * Convert a priority to a preemption level, taking the relevant BPR 632 * into account by zeroing the sub-priority bits. 633 */ 634 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp) 635 { 636 unsigned int bpr; 637 638 if (!grp) 639 bpr = __vgic_v3_get_bpr0(vmcr) + 1; 640 else 641 bpr = __vgic_v3_get_bpr1(vmcr); 642 643 return pri & (GENMASK(7, 0) << bpr); 644 } 645 646 /* 647 * The priority value is independent of any of the BPR values, so we 648 * normalize it using the minimal BPR value. This guarantees that no 649 * matter what the guest does with its BPR, we can always set/get the 650 * same value of a priority. 651 */ 652 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp) 653 { 654 u8 pre, ap; 655 u32 val; 656 int apr; 657 658 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp); 659 ap = pre >> __vgic_v3_bpr_min(); 660 apr = ap / 32; 661 662 if (!grp) { 663 val = __vgic_v3_read_ap0rn(apr); 664 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr); 665 } else { 666 val = __vgic_v3_read_ap1rn(apr); 667 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr); 668 } 669 } 670 671 static int __vgic_v3_clear_highest_active_priority(void) 672 { 673 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); 674 u32 hap = 0; 675 int i; 676 677 for (i = 0; i < nr_apr_regs; i++) { 678 u32 ap0, ap1; 679 int c0, c1; 680 681 ap0 = __vgic_v3_read_ap0rn(i); 682 ap1 = __vgic_v3_read_ap1rn(i); 683 if (!ap0 && !ap1) { 684 hap += 32; 685 continue; 686 } 687 688 c0 = ap0 ? __ffs(ap0) : 32; 689 c1 = ap1 ? __ffs(ap1) : 32; 690 691 /* Always clear the LSB, which is the highest priority */ 692 if (c0 < c1) { 693 ap0 &= ~BIT(c0); 694 __vgic_v3_write_ap0rn(ap0, i); 695 hap += c0; 696 } else { 697 ap1 &= ~BIT(c1); 698 __vgic_v3_write_ap1rn(ap1, i); 699 hap += c1; 700 } 701 702 /* Rescale to 8 bits of priority */ 703 return hap << __vgic_v3_bpr_min(); 704 } 705 706 return GICv3_IDLE_PRIORITY; 707 } 708 709 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 710 { 711 u64 lr_val; 712 u8 lr_prio, pmr; 713 int lr, grp; 714 715 grp = __vgic_v3_get_group(vcpu); 716 717 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); 718 if (lr < 0) 719 goto spurious; 720 721 if (grp != !!(lr_val & ICH_LR_GROUP)) 722 goto spurious; 723 724 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT; 725 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 726 if (pmr <= lr_prio) 727 goto spurious; 728 729 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp)) 730 goto spurious; 731 732 lr_val &= ~ICH_LR_STATE; 733 lr_val |= ICH_LR_ACTIVE_BIT; 734 __gic_v3_set_lr(lr_val, lr); 735 __vgic_v3_set_active_priority(lr_prio, vmcr, grp); 736 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); 737 return; 738 739 spurious: 740 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS); 741 } 742 743 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val) 744 { 745 lr_val &= ~ICH_LR_ACTIVE_BIT; 746 if (lr_val & ICH_LR_HW) { 747 u32 pid; 748 749 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT; 750 gic_write_dir(pid); 751 } 752 753 __gic_v3_set_lr(lr_val, lr); 754 } 755 756 static void __vgic_v3_bump_eoicount(void) 757 { 758 u32 hcr; 759 760 hcr = read_gicreg(ICH_HCR_EL2); 761 hcr += 1 << ICH_HCR_EL2_EOIcount_SHIFT; 762 write_gicreg(hcr, ICH_HCR_EL2); 763 } 764 765 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 766 { 767 u32 vid = vcpu_get_reg(vcpu, rt); 768 u64 lr_val; 769 int lr; 770 771 /* EOImode == 0, nothing to be done here */ 772 if (!(vmcr & ICH_VMCR_EOIM_MASK)) 773 return; 774 775 /* No deactivate to be performed on an LPI */ 776 if (vid >= VGIC_MIN_LPI) 777 return; 778 779 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); 780 if (lr == -1) { 781 __vgic_v3_bump_eoicount(); 782 return; 783 } 784 785 __vgic_v3_clear_active_lr(lr, lr_val); 786 } 787 788 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 789 { 790 u32 vid = vcpu_get_reg(vcpu, rt); 791 u64 lr_val; 792 u8 lr_prio, act_prio; 793 int lr, grp; 794 795 grp = __vgic_v3_get_group(vcpu); 796 797 /* Drop priority in any case */ 798 act_prio = __vgic_v3_clear_highest_active_priority(); 799 800 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); 801 if (lr == -1) { 802 /* Do not bump EOIcount for LPIs that aren't in the LRs */ 803 if (!(vid >= VGIC_MIN_LPI)) 804 __vgic_v3_bump_eoicount(); 805 return; 806 } 807 808 /* EOImode == 1 and not an LPI, nothing to be done here */ 809 if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI)) 810 return; 811 812 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 813 814 /* If priorities or group do not match, the guest has fscked-up. */ 815 if (grp != !!(lr_val & ICH_LR_GROUP) || 816 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio) 817 return; 818 819 /* Let's now perform the deactivation */ 820 __vgic_v3_clear_active_lr(lr, lr_val); 821 } 822 823 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 824 { 825 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK)); 826 } 827 828 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 829 { 830 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK)); 831 } 832 833 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 834 { 835 u64 val = vcpu_get_reg(vcpu, rt); 836 837 if (val & 1) 838 vmcr |= ICH_VMCR_ENG0_MASK; 839 else 840 vmcr &= ~ICH_VMCR_ENG0_MASK; 841 842 __vgic_v3_write_vmcr(vmcr); 843 } 844 845 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 846 { 847 u64 val = vcpu_get_reg(vcpu, rt); 848 849 if (val & 1) 850 vmcr |= ICH_VMCR_ENG1_MASK; 851 else 852 vmcr &= ~ICH_VMCR_ENG1_MASK; 853 854 __vgic_v3_write_vmcr(vmcr); 855 } 856 857 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 858 { 859 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr)); 860 } 861 862 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 863 { 864 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr)); 865 } 866 867 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 868 { 869 u64 val = vcpu_get_reg(vcpu, rt); 870 u8 bpr_min = __vgic_v3_bpr_min() - 1; 871 872 /* Enforce BPR limiting */ 873 if (val < bpr_min) 874 val = bpr_min; 875 876 val <<= ICH_VMCR_BPR0_SHIFT; 877 val &= ICH_VMCR_BPR0_MASK; 878 vmcr &= ~ICH_VMCR_BPR0_MASK; 879 vmcr |= val; 880 881 __vgic_v3_write_vmcr(vmcr); 882 } 883 884 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 885 { 886 u64 val = vcpu_get_reg(vcpu, rt); 887 u8 bpr_min = __vgic_v3_bpr_min(); 888 889 if (vmcr & ICH_VMCR_CBPR_MASK) 890 return; 891 892 /* Enforce BPR limiting */ 893 if (val < bpr_min) 894 val = bpr_min; 895 896 val <<= ICH_VMCR_BPR1_SHIFT; 897 val &= ICH_VMCR_BPR1_MASK; 898 vmcr &= ~ICH_VMCR_BPR1_MASK; 899 vmcr |= val; 900 901 __vgic_v3_write_vmcr(vmcr); 902 } 903 904 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n) 905 { 906 u32 val; 907 908 if (!__vgic_v3_get_group(vcpu)) 909 val = __vgic_v3_read_ap0rn(n); 910 else 911 val = __vgic_v3_read_ap1rn(n); 912 913 vcpu_set_reg(vcpu, rt, val); 914 } 915 916 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n) 917 { 918 u32 val = vcpu_get_reg(vcpu, rt); 919 920 if (!__vgic_v3_get_group(vcpu)) 921 __vgic_v3_write_ap0rn(val, n); 922 else 923 __vgic_v3_write_ap1rn(val, n); 924 } 925 926 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu, 927 u32 vmcr, int rt) 928 { 929 __vgic_v3_read_apxrn(vcpu, rt, 0); 930 } 931 932 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu, 933 u32 vmcr, int rt) 934 { 935 __vgic_v3_read_apxrn(vcpu, rt, 1); 936 } 937 938 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 939 { 940 __vgic_v3_read_apxrn(vcpu, rt, 2); 941 } 942 943 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 944 { 945 __vgic_v3_read_apxrn(vcpu, rt, 3); 946 } 947 948 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 949 { 950 __vgic_v3_write_apxrn(vcpu, rt, 0); 951 } 952 953 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 954 { 955 __vgic_v3_write_apxrn(vcpu, rt, 1); 956 } 957 958 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 959 { 960 __vgic_v3_write_apxrn(vcpu, rt, 2); 961 } 962 963 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 964 { 965 __vgic_v3_write_apxrn(vcpu, rt, 3); 966 } 967 968 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 969 { 970 u64 lr_val; 971 int lr, lr_grp, grp; 972 973 grp = __vgic_v3_get_group(vcpu); 974 975 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); 976 if (lr == -1) 977 goto spurious; 978 979 lr_grp = !!(lr_val & ICH_LR_GROUP); 980 if (lr_grp != grp) 981 lr_val = ICC_IAR1_EL1_SPURIOUS; 982 983 spurious: 984 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); 985 } 986 987 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 988 { 989 vmcr &= ICH_VMCR_PMR_MASK; 990 vmcr >>= ICH_VMCR_PMR_SHIFT; 991 vcpu_set_reg(vcpu, rt, vmcr); 992 } 993 994 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 995 { 996 u32 val = vcpu_get_reg(vcpu, rt); 997 998 val <<= ICH_VMCR_PMR_SHIFT; 999 val &= ICH_VMCR_PMR_MASK; 1000 vmcr &= ~ICH_VMCR_PMR_MASK; 1001 vmcr |= val; 1002 1003 write_gicreg(vmcr, ICH_VMCR_EL2); 1004 } 1005 1006 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1007 { 1008 u32 val = __vgic_v3_get_highest_active_priority(); 1009 vcpu_set_reg(vcpu, rt, val); 1010 } 1011 1012 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1013 { 1014 u32 vtr, val; 1015 1016 vtr = read_gicreg(ICH_VTR_EL2); 1017 /* PRIbits */ 1018 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT; 1019 /* IDbits */ 1020 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT; 1021 /* A3V */ 1022 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT; 1023 /* EOImode */ 1024 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT; 1025 /* CBPR */ 1026 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT; 1027 1028 vcpu_set_reg(vcpu, rt, val); 1029 } 1030 1031 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1032 { 1033 u32 val = vcpu_get_reg(vcpu, rt); 1034 1035 if (val & ICC_CTLR_EL1_CBPR_MASK) 1036 vmcr |= ICH_VMCR_CBPR_MASK; 1037 else 1038 vmcr &= ~ICH_VMCR_CBPR_MASK; 1039 1040 if (val & ICC_CTLR_EL1_EOImode_MASK) 1041 vmcr |= ICH_VMCR_EOIM_MASK; 1042 else 1043 vmcr &= ~ICH_VMCR_EOIM_MASK; 1044 1045 write_gicreg(vmcr, ICH_VMCR_EL2); 1046 } 1047 1048 static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu, 1049 u32 sysreg, bool is_read) 1050 { 1051 u64 ich_hcr; 1052 1053 if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu)) 1054 return false; 1055 1056 ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2); 1057 1058 switch (sysreg) { 1059 case SYS_ICC_IGRPEN0_EL1: 1060 if (is_read && 1061 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1)) 1062 return true; 1063 1064 if (!is_read && 1065 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1)) 1066 return true; 1067 1068 fallthrough; 1069 1070 case SYS_ICC_AP0Rn_EL1(0): 1071 case SYS_ICC_AP0Rn_EL1(1): 1072 case SYS_ICC_AP0Rn_EL1(2): 1073 case SYS_ICC_AP0Rn_EL1(3): 1074 case SYS_ICC_BPR0_EL1: 1075 case SYS_ICC_EOIR0_EL1: 1076 case SYS_ICC_HPPIR0_EL1: 1077 case SYS_ICC_IAR0_EL1: 1078 return ich_hcr & ICH_HCR_EL2_TALL0; 1079 1080 case SYS_ICC_IGRPEN1_EL1: 1081 if (is_read && 1082 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1)) 1083 return true; 1084 1085 if (!is_read && 1086 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1)) 1087 return true; 1088 1089 fallthrough; 1090 1091 case SYS_ICC_AP1Rn_EL1(0): 1092 case SYS_ICC_AP1Rn_EL1(1): 1093 case SYS_ICC_AP1Rn_EL1(2): 1094 case SYS_ICC_AP1Rn_EL1(3): 1095 case SYS_ICC_BPR1_EL1: 1096 case SYS_ICC_EOIR1_EL1: 1097 case SYS_ICC_HPPIR1_EL1: 1098 case SYS_ICC_IAR1_EL1: 1099 return ich_hcr & ICH_HCR_EL2_TALL1; 1100 1101 case SYS_ICC_DIR_EL1: 1102 if (ich_hcr & ICH_HCR_EL2_TDIR) 1103 return true; 1104 1105 fallthrough; 1106 1107 case SYS_ICC_RPR_EL1: 1108 case SYS_ICC_CTLR_EL1: 1109 case SYS_ICC_PMR_EL1: 1110 return ich_hcr & ICH_HCR_EL2_TC; 1111 1112 default: 1113 return false; 1114 } 1115 } 1116 1117 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu) 1118 { 1119 int rt; 1120 u64 esr; 1121 u32 vmcr; 1122 void (*fn)(struct kvm_vcpu *, u32, int); 1123 bool is_read; 1124 u32 sysreg; 1125 1126 if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3) 1127 return 0; 1128 1129 esr = kvm_vcpu_get_esr(vcpu); 1130 if (vcpu_mode_is_32bit(vcpu)) { 1131 if (!kvm_condition_valid(vcpu)) { 1132 __kvm_skip_instr(vcpu); 1133 return 1; 1134 } 1135 1136 sysreg = esr_cp15_to_sysreg(esr); 1137 } else { 1138 sysreg = esr_sys64_to_sysreg(esr); 1139 } 1140 1141 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ; 1142 1143 if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read)) 1144 return 0; 1145 1146 switch (sysreg) { 1147 case SYS_ICC_IAR0_EL1: 1148 case SYS_ICC_IAR1_EL1: 1149 if (unlikely(!is_read)) 1150 return 0; 1151 fn = __vgic_v3_read_iar; 1152 break; 1153 case SYS_ICC_EOIR0_EL1: 1154 case SYS_ICC_EOIR1_EL1: 1155 if (unlikely(is_read)) 1156 return 0; 1157 fn = __vgic_v3_write_eoir; 1158 break; 1159 case SYS_ICC_IGRPEN1_EL1: 1160 if (is_read) 1161 fn = __vgic_v3_read_igrpen1; 1162 else 1163 fn = __vgic_v3_write_igrpen1; 1164 break; 1165 case SYS_ICC_BPR1_EL1: 1166 if (is_read) 1167 fn = __vgic_v3_read_bpr1; 1168 else 1169 fn = __vgic_v3_write_bpr1; 1170 break; 1171 case SYS_ICC_AP0Rn_EL1(0): 1172 case SYS_ICC_AP1Rn_EL1(0): 1173 if (is_read) 1174 fn = __vgic_v3_read_apxr0; 1175 else 1176 fn = __vgic_v3_write_apxr0; 1177 break; 1178 case SYS_ICC_AP0Rn_EL1(1): 1179 case SYS_ICC_AP1Rn_EL1(1): 1180 if (is_read) 1181 fn = __vgic_v3_read_apxr1; 1182 else 1183 fn = __vgic_v3_write_apxr1; 1184 break; 1185 case SYS_ICC_AP0Rn_EL1(2): 1186 case SYS_ICC_AP1Rn_EL1(2): 1187 if (is_read) 1188 fn = __vgic_v3_read_apxr2; 1189 else 1190 fn = __vgic_v3_write_apxr2; 1191 break; 1192 case SYS_ICC_AP0Rn_EL1(3): 1193 case SYS_ICC_AP1Rn_EL1(3): 1194 if (is_read) 1195 fn = __vgic_v3_read_apxr3; 1196 else 1197 fn = __vgic_v3_write_apxr3; 1198 break; 1199 case SYS_ICC_HPPIR0_EL1: 1200 case SYS_ICC_HPPIR1_EL1: 1201 if (unlikely(!is_read)) 1202 return 0; 1203 fn = __vgic_v3_read_hppir; 1204 break; 1205 case SYS_ICC_IGRPEN0_EL1: 1206 if (is_read) 1207 fn = __vgic_v3_read_igrpen0; 1208 else 1209 fn = __vgic_v3_write_igrpen0; 1210 break; 1211 case SYS_ICC_BPR0_EL1: 1212 if (is_read) 1213 fn = __vgic_v3_read_bpr0; 1214 else 1215 fn = __vgic_v3_write_bpr0; 1216 break; 1217 case SYS_ICC_DIR_EL1: 1218 if (unlikely(is_read)) 1219 return 0; 1220 fn = __vgic_v3_write_dir; 1221 break; 1222 case SYS_ICC_RPR_EL1: 1223 if (unlikely(!is_read)) 1224 return 0; 1225 fn = __vgic_v3_read_rpr; 1226 break; 1227 case SYS_ICC_CTLR_EL1: 1228 if (is_read) 1229 fn = __vgic_v3_read_ctlr; 1230 else 1231 fn = __vgic_v3_write_ctlr; 1232 break; 1233 case SYS_ICC_PMR_EL1: 1234 if (is_read) 1235 fn = __vgic_v3_read_pmr; 1236 else 1237 fn = __vgic_v3_write_pmr; 1238 break; 1239 default: 1240 return 0; 1241 } 1242 1243 vmcr = __vgic_v3_read_vmcr(); 1244 rt = kvm_vcpu_sys_get_rt(vcpu); 1245 fn(vcpu, vmcr, rt); 1246 1247 __kvm_skip_instr(vcpu); 1248 1249 return 1; 1250 } 1251