1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012-2015 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <hyp/adjust_pc.h> 8 9 #include <linux/compiler.h> 10 #include <linux/irqchip/arm-gic-v3.h> 11 #include <linux/kvm_host.h> 12 13 #include <asm/kvm_emulate.h> 14 #include <asm/kvm_hyp.h> 15 #include <asm/kvm_mmu.h> 16 17 #define vtr_to_max_lr_idx(v) ((v) & 0xf) 18 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1) 19 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5)) 20 21 static u64 __gic_v3_get_lr(unsigned int lr) 22 { 23 switch (lr & 0xf) { 24 case 0: 25 return read_gicreg(ICH_LR0_EL2); 26 case 1: 27 return read_gicreg(ICH_LR1_EL2); 28 case 2: 29 return read_gicreg(ICH_LR2_EL2); 30 case 3: 31 return read_gicreg(ICH_LR3_EL2); 32 case 4: 33 return read_gicreg(ICH_LR4_EL2); 34 case 5: 35 return read_gicreg(ICH_LR5_EL2); 36 case 6: 37 return read_gicreg(ICH_LR6_EL2); 38 case 7: 39 return read_gicreg(ICH_LR7_EL2); 40 case 8: 41 return read_gicreg(ICH_LR8_EL2); 42 case 9: 43 return read_gicreg(ICH_LR9_EL2); 44 case 10: 45 return read_gicreg(ICH_LR10_EL2); 46 case 11: 47 return read_gicreg(ICH_LR11_EL2); 48 case 12: 49 return read_gicreg(ICH_LR12_EL2); 50 case 13: 51 return read_gicreg(ICH_LR13_EL2); 52 case 14: 53 return read_gicreg(ICH_LR14_EL2); 54 case 15: 55 return read_gicreg(ICH_LR15_EL2); 56 } 57 58 unreachable(); 59 } 60 61 static void __gic_v3_set_lr(u64 val, int lr) 62 { 63 switch (lr & 0xf) { 64 case 0: 65 write_gicreg(val, ICH_LR0_EL2); 66 break; 67 case 1: 68 write_gicreg(val, ICH_LR1_EL2); 69 break; 70 case 2: 71 write_gicreg(val, ICH_LR2_EL2); 72 break; 73 case 3: 74 write_gicreg(val, ICH_LR3_EL2); 75 break; 76 case 4: 77 write_gicreg(val, ICH_LR4_EL2); 78 break; 79 case 5: 80 write_gicreg(val, ICH_LR5_EL2); 81 break; 82 case 6: 83 write_gicreg(val, ICH_LR6_EL2); 84 break; 85 case 7: 86 write_gicreg(val, ICH_LR7_EL2); 87 break; 88 case 8: 89 write_gicreg(val, ICH_LR8_EL2); 90 break; 91 case 9: 92 write_gicreg(val, ICH_LR9_EL2); 93 break; 94 case 10: 95 write_gicreg(val, ICH_LR10_EL2); 96 break; 97 case 11: 98 write_gicreg(val, ICH_LR11_EL2); 99 break; 100 case 12: 101 write_gicreg(val, ICH_LR12_EL2); 102 break; 103 case 13: 104 write_gicreg(val, ICH_LR13_EL2); 105 break; 106 case 14: 107 write_gicreg(val, ICH_LR14_EL2); 108 break; 109 case 15: 110 write_gicreg(val, ICH_LR15_EL2); 111 break; 112 } 113 } 114 115 static void __vgic_v3_write_ap0rn(u32 val, int n) 116 { 117 switch (n) { 118 case 0: 119 write_gicreg(val, ICH_AP0R0_EL2); 120 break; 121 case 1: 122 write_gicreg(val, ICH_AP0R1_EL2); 123 break; 124 case 2: 125 write_gicreg(val, ICH_AP0R2_EL2); 126 break; 127 case 3: 128 write_gicreg(val, ICH_AP0R3_EL2); 129 break; 130 } 131 } 132 133 static void __vgic_v3_write_ap1rn(u32 val, int n) 134 { 135 switch (n) { 136 case 0: 137 write_gicreg(val, ICH_AP1R0_EL2); 138 break; 139 case 1: 140 write_gicreg(val, ICH_AP1R1_EL2); 141 break; 142 case 2: 143 write_gicreg(val, ICH_AP1R2_EL2); 144 break; 145 case 3: 146 write_gicreg(val, ICH_AP1R3_EL2); 147 break; 148 } 149 } 150 151 static u32 __vgic_v3_read_ap0rn(int n) 152 { 153 u32 val; 154 155 switch (n) { 156 case 0: 157 val = read_gicreg(ICH_AP0R0_EL2); 158 break; 159 case 1: 160 val = read_gicreg(ICH_AP0R1_EL2); 161 break; 162 case 2: 163 val = read_gicreg(ICH_AP0R2_EL2); 164 break; 165 case 3: 166 val = read_gicreg(ICH_AP0R3_EL2); 167 break; 168 default: 169 unreachable(); 170 } 171 172 return val; 173 } 174 175 static u32 __vgic_v3_read_ap1rn(int n) 176 { 177 u32 val; 178 179 switch (n) { 180 case 0: 181 val = read_gicreg(ICH_AP1R0_EL2); 182 break; 183 case 1: 184 val = read_gicreg(ICH_AP1R1_EL2); 185 break; 186 case 2: 187 val = read_gicreg(ICH_AP1R2_EL2); 188 break; 189 case 3: 190 val = read_gicreg(ICH_AP1R3_EL2); 191 break; 192 default: 193 unreachable(); 194 } 195 196 return val; 197 } 198 199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if) 200 { 201 u64 used_lrs = cpu_if->used_lrs; 202 203 /* 204 * Make sure stores to the GIC via the memory mapped interface 205 * are now visible to the system register interface when reading the 206 * LRs, and when reading back the VMCR on non-VHE systems. 207 */ 208 if (used_lrs || !has_vhe()) { 209 if (!cpu_if->vgic_sre) { 210 dsb(sy); 211 isb(); 212 } 213 } 214 215 if (used_lrs || cpu_if->its_vpe.its_vm) { 216 int i; 217 u32 elrsr; 218 219 elrsr = read_gicreg(ICH_ELRSR_EL2); 220 221 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2); 222 223 for (i = 0; i < used_lrs; i++) { 224 if (elrsr & (1 << i)) 225 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE; 226 else 227 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i); 228 229 __gic_v3_set_lr(0, i); 230 } 231 } 232 } 233 234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if) 235 { 236 u64 used_lrs = cpu_if->used_lrs; 237 int i; 238 239 if (used_lrs || cpu_if->its_vpe.its_vm) { 240 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); 241 242 for (i = 0; i < used_lrs; i++) 243 __gic_v3_set_lr(cpu_if->vgic_lr[i], i); 244 } 245 246 /* 247 * Ensure that writes to the LRs, and on non-VHE systems ensure that 248 * the write to the VMCR in __vgic_v3_activate_traps(), will have 249 * reached the (re)distributors. This ensure the guest will read the 250 * correct values from the memory-mapped interface. 251 */ 252 if (used_lrs || !has_vhe()) { 253 if (!cpu_if->vgic_sre) { 254 isb(); 255 dsb(sy); 256 } 257 } 258 } 259 260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if) 261 { 262 /* 263 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a 264 * Group0 interrupt (as generated in GICv2 mode) to be 265 * delivered as a FIQ to the guest, with potentially fatal 266 * consequences. So we must make sure that ICC_SRE_EL1 has 267 * been actually programmed with the value we want before 268 * starting to mess with the rest of the GIC, and VMCR_EL2 in 269 * particular. This logic must be called before 270 * __vgic_v3_restore_state(). 271 */ 272 if (!cpu_if->vgic_sre) { 273 write_gicreg(0, ICC_SRE_EL1); 274 isb(); 275 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2); 276 277 278 if (has_vhe()) { 279 /* 280 * Ensure that the write to the VMCR will have reached 281 * the (re)distributors. This ensure the guest will 282 * read the correct values from the memory-mapped 283 * interface. 284 */ 285 isb(); 286 dsb(sy); 287 } 288 } 289 290 /* 291 * Prevent the guest from touching the GIC system registers if 292 * SRE isn't enabled for GICv3 emulation. 293 */ 294 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE, 295 ICC_SRE_EL2); 296 297 /* 298 * If we need to trap system registers, we must write 299 * ICH_HCR_EL2 anyway, even if no interrupts are being 300 * injected, 301 */ 302 if (static_branch_unlikely(&vgic_v3_cpuif_trap) || 303 cpu_if->its_vpe.its_vm) 304 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2); 305 } 306 307 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if) 308 { 309 u64 val; 310 311 if (!cpu_if->vgic_sre) { 312 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2); 313 } 314 315 val = read_gicreg(ICC_SRE_EL2); 316 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2); 317 318 if (!cpu_if->vgic_sre) { 319 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */ 320 isb(); 321 write_gicreg(1, ICC_SRE_EL1); 322 } 323 324 /* 325 * If we were trapping system registers, we enabled the VGIC even if 326 * no interrupts were being injected, and we disable it again here. 327 */ 328 if (static_branch_unlikely(&vgic_v3_cpuif_trap) || 329 cpu_if->its_vpe.its_vm) 330 write_gicreg(0, ICH_HCR_EL2); 331 } 332 333 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if) 334 { 335 u64 val; 336 u32 nr_pre_bits; 337 338 val = read_gicreg(ICH_VTR_EL2); 339 nr_pre_bits = vtr_to_nr_pre_bits(val); 340 341 switch (nr_pre_bits) { 342 case 7: 343 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3); 344 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2); 345 fallthrough; 346 case 6: 347 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1); 348 fallthrough; 349 default: 350 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0); 351 } 352 353 switch (nr_pre_bits) { 354 case 7: 355 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3); 356 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2); 357 fallthrough; 358 case 6: 359 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1); 360 fallthrough; 361 default: 362 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0); 363 } 364 } 365 366 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) 367 { 368 u64 val; 369 u32 nr_pre_bits; 370 371 val = read_gicreg(ICH_VTR_EL2); 372 nr_pre_bits = vtr_to_nr_pre_bits(val); 373 374 switch (nr_pre_bits) { 375 case 7: 376 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3); 377 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2); 378 fallthrough; 379 case 6: 380 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1); 381 fallthrough; 382 default: 383 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0); 384 } 385 386 switch (nr_pre_bits) { 387 case 7: 388 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3); 389 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2); 390 fallthrough; 391 case 6: 392 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1); 393 fallthrough; 394 default: 395 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0); 396 } 397 } 398 399 void __vgic_v3_init_lrs(void) 400 { 401 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2)); 402 int i; 403 404 for (i = 0; i <= max_lr_idx; i++) 405 __gic_v3_set_lr(0, i); 406 } 407 408 /* 409 * Return the GIC CPU configuration: 410 * - [31:0] ICH_VTR_EL2 411 * - [62:32] RES0 412 * - [63] MMIO (GICv2) capable 413 */ 414 u64 __vgic_v3_get_gic_config(void) 415 { 416 u64 val, sre = read_gicreg(ICC_SRE_EL1); 417 unsigned long flags = 0; 418 419 /* 420 * To check whether we have a MMIO-based (GICv2 compatible) 421 * CPU interface, we need to disable the system register 422 * view. To do that safely, we have to prevent any interrupt 423 * from firing (which would be deadly). 424 * 425 * Note that this only makes sense on VHE, as interrupts are 426 * already masked for nVHE as part of the exception entry to 427 * EL2. 428 */ 429 if (has_vhe()) 430 flags = local_daif_save(); 431 432 /* 433 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates 434 * that to be able to set ICC_SRE_EL1.SRE to 0, all the 435 * interrupt overrides must be set. You've got to love this. 436 */ 437 sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO); 438 isb(); 439 write_gicreg(0, ICC_SRE_EL1); 440 isb(); 441 442 val = read_gicreg(ICC_SRE_EL1); 443 444 write_gicreg(sre, ICC_SRE_EL1); 445 isb(); 446 sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0); 447 isb(); 448 449 if (has_vhe()) 450 local_daif_restore(flags); 451 452 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); 453 val |= read_gicreg(ICH_VTR_EL2); 454 455 return val; 456 } 457 458 static u64 __vgic_v3_read_vmcr(void) 459 { 460 return read_gicreg(ICH_VMCR_EL2); 461 } 462 463 static void __vgic_v3_write_vmcr(u32 vmcr) 464 { 465 write_gicreg(vmcr, ICH_VMCR_EL2); 466 } 467 468 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) 469 { 470 __vgic_v3_save_aprs(cpu_if); 471 if (cpu_if->vgic_sre) 472 cpu_if->vgic_vmcr = __vgic_v3_read_vmcr(); 473 } 474 475 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) 476 { 477 /* 478 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen 479 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the 480 * VMCR_EL2 save/restore in the world switch. 481 */ 482 if (cpu_if->vgic_sre) 483 __vgic_v3_write_vmcr(cpu_if->vgic_vmcr); 484 __vgic_v3_restore_aprs(cpu_if); 485 } 486 487 static int __vgic_v3_bpr_min(void) 488 { 489 /* See Pseudocode for VPriorityGroup */ 490 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2)); 491 } 492 493 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu) 494 { 495 u64 esr = kvm_vcpu_get_esr(vcpu); 496 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT; 497 498 return crm != 8; 499 } 500 501 #define GICv3_IDLE_PRIORITY 0xff 502 503 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr, 504 u64 *lr_val) 505 { 506 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; 507 u8 priority = GICv3_IDLE_PRIORITY; 508 int i, lr = -1; 509 510 for (i = 0; i < used_lrs; i++) { 511 u64 val = __gic_v3_get_lr(i); 512 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 513 514 /* Not pending in the state? */ 515 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT) 516 continue; 517 518 /* Group-0 interrupt, but Group-0 disabled? */ 519 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK)) 520 continue; 521 522 /* Group-1 interrupt, but Group-1 disabled? */ 523 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK)) 524 continue; 525 526 /* Not the highest priority? */ 527 if (lr_prio >= priority) 528 continue; 529 530 /* This is a candidate */ 531 priority = lr_prio; 532 *lr_val = val; 533 lr = i; 534 } 535 536 if (lr == -1) 537 *lr_val = ICC_IAR1_EL1_SPURIOUS; 538 539 return lr; 540 } 541 542 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid, 543 u64 *lr_val) 544 { 545 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs; 546 int i; 547 548 for (i = 0; i < used_lrs; i++) { 549 u64 val = __gic_v3_get_lr(i); 550 551 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid && 552 (val & ICH_LR_ACTIVE_BIT)) { 553 *lr_val = val; 554 return i; 555 } 556 } 557 558 *lr_val = ICC_IAR1_EL1_SPURIOUS; 559 return -1; 560 } 561 562 static int __vgic_v3_get_highest_active_priority(void) 563 { 564 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); 565 u32 hap = 0; 566 int i; 567 568 for (i = 0; i < nr_apr_regs; i++) { 569 u32 val; 570 571 /* 572 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers 573 * contain the active priority levels for this VCPU 574 * for the maximum number of supported priority 575 * levels, and we return the full priority level only 576 * if the BPR is programmed to its minimum, otherwise 577 * we return a combination of the priority level and 578 * subpriority, as determined by the setting of the 579 * BPR, but without the full subpriority. 580 */ 581 val = __vgic_v3_read_ap0rn(i); 582 val |= __vgic_v3_read_ap1rn(i); 583 if (!val) { 584 hap += 32; 585 continue; 586 } 587 588 return (hap + __ffs(val)) << __vgic_v3_bpr_min(); 589 } 590 591 return GICv3_IDLE_PRIORITY; 592 } 593 594 static unsigned int __vgic_v3_get_bpr0(u32 vmcr) 595 { 596 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT; 597 } 598 599 static unsigned int __vgic_v3_get_bpr1(u32 vmcr) 600 { 601 unsigned int bpr; 602 603 if (vmcr & ICH_VMCR_CBPR_MASK) { 604 bpr = __vgic_v3_get_bpr0(vmcr); 605 if (bpr < 7) 606 bpr++; 607 } else { 608 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT; 609 } 610 611 return bpr; 612 } 613 614 /* 615 * Convert a priority to a preemption level, taking the relevant BPR 616 * into account by zeroing the sub-priority bits. 617 */ 618 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp) 619 { 620 unsigned int bpr; 621 622 if (!grp) 623 bpr = __vgic_v3_get_bpr0(vmcr) + 1; 624 else 625 bpr = __vgic_v3_get_bpr1(vmcr); 626 627 return pri & (GENMASK(7, 0) << bpr); 628 } 629 630 /* 631 * The priority value is independent of any of the BPR values, so we 632 * normalize it using the minimal BPR value. This guarantees that no 633 * matter what the guest does with its BPR, we can always set/get the 634 * same value of a priority. 635 */ 636 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp) 637 { 638 u8 pre, ap; 639 u32 val; 640 int apr; 641 642 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp); 643 ap = pre >> __vgic_v3_bpr_min(); 644 apr = ap / 32; 645 646 if (!grp) { 647 val = __vgic_v3_read_ap0rn(apr); 648 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr); 649 } else { 650 val = __vgic_v3_read_ap1rn(apr); 651 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr); 652 } 653 } 654 655 static int __vgic_v3_clear_highest_active_priority(void) 656 { 657 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); 658 u32 hap = 0; 659 int i; 660 661 for (i = 0; i < nr_apr_regs; i++) { 662 u32 ap0, ap1; 663 int c0, c1; 664 665 ap0 = __vgic_v3_read_ap0rn(i); 666 ap1 = __vgic_v3_read_ap1rn(i); 667 if (!ap0 && !ap1) { 668 hap += 32; 669 continue; 670 } 671 672 c0 = ap0 ? __ffs(ap0) : 32; 673 c1 = ap1 ? __ffs(ap1) : 32; 674 675 /* Always clear the LSB, which is the highest priority */ 676 if (c0 < c1) { 677 ap0 &= ~BIT(c0); 678 __vgic_v3_write_ap0rn(ap0, i); 679 hap += c0; 680 } else { 681 ap1 &= ~BIT(c1); 682 __vgic_v3_write_ap1rn(ap1, i); 683 hap += c1; 684 } 685 686 /* Rescale to 8 bits of priority */ 687 return hap << __vgic_v3_bpr_min(); 688 } 689 690 return GICv3_IDLE_PRIORITY; 691 } 692 693 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 694 { 695 u64 lr_val; 696 u8 lr_prio, pmr; 697 int lr, grp; 698 699 grp = __vgic_v3_get_group(vcpu); 700 701 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); 702 if (lr < 0) 703 goto spurious; 704 705 if (grp != !!(lr_val & ICH_LR_GROUP)) 706 goto spurious; 707 708 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT; 709 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 710 if (pmr <= lr_prio) 711 goto spurious; 712 713 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp)) 714 goto spurious; 715 716 lr_val &= ~ICH_LR_STATE; 717 lr_val |= ICH_LR_ACTIVE_BIT; 718 __gic_v3_set_lr(lr_val, lr); 719 __vgic_v3_set_active_priority(lr_prio, vmcr, grp); 720 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); 721 return; 722 723 spurious: 724 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS); 725 } 726 727 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val) 728 { 729 lr_val &= ~ICH_LR_ACTIVE_BIT; 730 if (lr_val & ICH_LR_HW) { 731 u32 pid; 732 733 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT; 734 gic_write_dir(pid); 735 } 736 737 __gic_v3_set_lr(lr_val, lr); 738 } 739 740 static void __vgic_v3_bump_eoicount(void) 741 { 742 u32 hcr; 743 744 hcr = read_gicreg(ICH_HCR_EL2); 745 hcr += 1 << ICH_HCR_EOIcount_SHIFT; 746 write_gicreg(hcr, ICH_HCR_EL2); 747 } 748 749 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 750 { 751 u32 vid = vcpu_get_reg(vcpu, rt); 752 u64 lr_val; 753 int lr; 754 755 /* EOImode == 0, nothing to be done here */ 756 if (!(vmcr & ICH_VMCR_EOIM_MASK)) 757 return; 758 759 /* No deactivate to be performed on an LPI */ 760 if (vid >= VGIC_MIN_LPI) 761 return; 762 763 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); 764 if (lr == -1) { 765 __vgic_v3_bump_eoicount(); 766 return; 767 } 768 769 __vgic_v3_clear_active_lr(lr, lr_val); 770 } 771 772 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 773 { 774 u32 vid = vcpu_get_reg(vcpu, rt); 775 u64 lr_val; 776 u8 lr_prio, act_prio; 777 int lr, grp; 778 779 grp = __vgic_v3_get_group(vcpu); 780 781 /* Drop priority in any case */ 782 act_prio = __vgic_v3_clear_highest_active_priority(); 783 784 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val); 785 if (lr == -1) { 786 /* Do not bump EOIcount for LPIs that aren't in the LRs */ 787 if (!(vid >= VGIC_MIN_LPI)) 788 __vgic_v3_bump_eoicount(); 789 return; 790 } 791 792 /* EOImode == 1 and not an LPI, nothing to be done here */ 793 if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI)) 794 return; 795 796 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT; 797 798 /* If priorities or group do not match, the guest has fscked-up. */ 799 if (grp != !!(lr_val & ICH_LR_GROUP) || 800 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio) 801 return; 802 803 /* Let's now perform the deactivation */ 804 __vgic_v3_clear_active_lr(lr, lr_val); 805 } 806 807 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 808 { 809 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK)); 810 } 811 812 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 813 { 814 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK)); 815 } 816 817 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 818 { 819 u64 val = vcpu_get_reg(vcpu, rt); 820 821 if (val & 1) 822 vmcr |= ICH_VMCR_ENG0_MASK; 823 else 824 vmcr &= ~ICH_VMCR_ENG0_MASK; 825 826 __vgic_v3_write_vmcr(vmcr); 827 } 828 829 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 830 { 831 u64 val = vcpu_get_reg(vcpu, rt); 832 833 if (val & 1) 834 vmcr |= ICH_VMCR_ENG1_MASK; 835 else 836 vmcr &= ~ICH_VMCR_ENG1_MASK; 837 838 __vgic_v3_write_vmcr(vmcr); 839 } 840 841 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 842 { 843 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr)); 844 } 845 846 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 847 { 848 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr)); 849 } 850 851 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 852 { 853 u64 val = vcpu_get_reg(vcpu, rt); 854 u8 bpr_min = __vgic_v3_bpr_min() - 1; 855 856 /* Enforce BPR limiting */ 857 if (val < bpr_min) 858 val = bpr_min; 859 860 val <<= ICH_VMCR_BPR0_SHIFT; 861 val &= ICH_VMCR_BPR0_MASK; 862 vmcr &= ~ICH_VMCR_BPR0_MASK; 863 vmcr |= val; 864 865 __vgic_v3_write_vmcr(vmcr); 866 } 867 868 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 869 { 870 u64 val = vcpu_get_reg(vcpu, rt); 871 u8 bpr_min = __vgic_v3_bpr_min(); 872 873 if (vmcr & ICH_VMCR_CBPR_MASK) 874 return; 875 876 /* Enforce BPR limiting */ 877 if (val < bpr_min) 878 val = bpr_min; 879 880 val <<= ICH_VMCR_BPR1_SHIFT; 881 val &= ICH_VMCR_BPR1_MASK; 882 vmcr &= ~ICH_VMCR_BPR1_MASK; 883 vmcr |= val; 884 885 __vgic_v3_write_vmcr(vmcr); 886 } 887 888 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n) 889 { 890 u32 val; 891 892 if (!__vgic_v3_get_group(vcpu)) 893 val = __vgic_v3_read_ap0rn(n); 894 else 895 val = __vgic_v3_read_ap1rn(n); 896 897 vcpu_set_reg(vcpu, rt, val); 898 } 899 900 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n) 901 { 902 u32 val = vcpu_get_reg(vcpu, rt); 903 904 if (!__vgic_v3_get_group(vcpu)) 905 __vgic_v3_write_ap0rn(val, n); 906 else 907 __vgic_v3_write_ap1rn(val, n); 908 } 909 910 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu, 911 u32 vmcr, int rt) 912 { 913 __vgic_v3_read_apxrn(vcpu, rt, 0); 914 } 915 916 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu, 917 u32 vmcr, int rt) 918 { 919 __vgic_v3_read_apxrn(vcpu, rt, 1); 920 } 921 922 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 923 { 924 __vgic_v3_read_apxrn(vcpu, rt, 2); 925 } 926 927 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 928 { 929 __vgic_v3_read_apxrn(vcpu, rt, 3); 930 } 931 932 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 933 { 934 __vgic_v3_write_apxrn(vcpu, rt, 0); 935 } 936 937 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 938 { 939 __vgic_v3_write_apxrn(vcpu, rt, 1); 940 } 941 942 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 943 { 944 __vgic_v3_write_apxrn(vcpu, rt, 2); 945 } 946 947 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 948 { 949 __vgic_v3_write_apxrn(vcpu, rt, 3); 950 } 951 952 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 953 { 954 u64 lr_val; 955 int lr, lr_grp, grp; 956 957 grp = __vgic_v3_get_group(vcpu); 958 959 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val); 960 if (lr == -1) 961 goto spurious; 962 963 lr_grp = !!(lr_val & ICH_LR_GROUP); 964 if (lr_grp != grp) 965 lr_val = ICC_IAR1_EL1_SPURIOUS; 966 967 spurious: 968 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK); 969 } 970 971 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 972 { 973 vmcr &= ICH_VMCR_PMR_MASK; 974 vmcr >>= ICH_VMCR_PMR_SHIFT; 975 vcpu_set_reg(vcpu, rt, vmcr); 976 } 977 978 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 979 { 980 u32 val = vcpu_get_reg(vcpu, rt); 981 982 val <<= ICH_VMCR_PMR_SHIFT; 983 val &= ICH_VMCR_PMR_MASK; 984 vmcr &= ~ICH_VMCR_PMR_MASK; 985 vmcr |= val; 986 987 write_gicreg(vmcr, ICH_VMCR_EL2); 988 } 989 990 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 991 { 992 u32 val = __vgic_v3_get_highest_active_priority(); 993 vcpu_set_reg(vcpu, rt, val); 994 } 995 996 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 997 { 998 u32 vtr, val; 999 1000 vtr = read_gicreg(ICH_VTR_EL2); 1001 /* PRIbits */ 1002 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT; 1003 /* IDbits */ 1004 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT; 1005 /* SEIS */ 1006 if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK) 1007 val |= BIT(ICC_CTLR_EL1_SEIS_SHIFT); 1008 /* A3V */ 1009 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT; 1010 /* EOImode */ 1011 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT; 1012 /* CBPR */ 1013 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT; 1014 1015 vcpu_set_reg(vcpu, rt, val); 1016 } 1017 1018 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) 1019 { 1020 u32 val = vcpu_get_reg(vcpu, rt); 1021 1022 if (val & ICC_CTLR_EL1_CBPR_MASK) 1023 vmcr |= ICH_VMCR_CBPR_MASK; 1024 else 1025 vmcr &= ~ICH_VMCR_CBPR_MASK; 1026 1027 if (val & ICC_CTLR_EL1_EOImode_MASK) 1028 vmcr |= ICH_VMCR_EOIM_MASK; 1029 else 1030 vmcr &= ~ICH_VMCR_EOIM_MASK; 1031 1032 write_gicreg(vmcr, ICH_VMCR_EL2); 1033 } 1034 1035 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu) 1036 { 1037 int rt; 1038 u64 esr; 1039 u32 vmcr; 1040 void (*fn)(struct kvm_vcpu *, u32, int); 1041 bool is_read; 1042 u32 sysreg; 1043 1044 esr = kvm_vcpu_get_esr(vcpu); 1045 if (vcpu_mode_is_32bit(vcpu)) { 1046 if (!kvm_condition_valid(vcpu)) { 1047 __kvm_skip_instr(vcpu); 1048 return 1; 1049 } 1050 1051 sysreg = esr_cp15_to_sysreg(esr); 1052 } else { 1053 sysreg = esr_sys64_to_sysreg(esr); 1054 } 1055 1056 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ; 1057 1058 switch (sysreg) { 1059 case SYS_ICC_IAR0_EL1: 1060 case SYS_ICC_IAR1_EL1: 1061 if (unlikely(!is_read)) 1062 return 0; 1063 fn = __vgic_v3_read_iar; 1064 break; 1065 case SYS_ICC_EOIR0_EL1: 1066 case SYS_ICC_EOIR1_EL1: 1067 if (unlikely(is_read)) 1068 return 0; 1069 fn = __vgic_v3_write_eoir; 1070 break; 1071 case SYS_ICC_IGRPEN1_EL1: 1072 if (is_read) 1073 fn = __vgic_v3_read_igrpen1; 1074 else 1075 fn = __vgic_v3_write_igrpen1; 1076 break; 1077 case SYS_ICC_BPR1_EL1: 1078 if (is_read) 1079 fn = __vgic_v3_read_bpr1; 1080 else 1081 fn = __vgic_v3_write_bpr1; 1082 break; 1083 case SYS_ICC_AP0Rn_EL1(0): 1084 case SYS_ICC_AP1Rn_EL1(0): 1085 if (is_read) 1086 fn = __vgic_v3_read_apxr0; 1087 else 1088 fn = __vgic_v3_write_apxr0; 1089 break; 1090 case SYS_ICC_AP0Rn_EL1(1): 1091 case SYS_ICC_AP1Rn_EL1(1): 1092 if (is_read) 1093 fn = __vgic_v3_read_apxr1; 1094 else 1095 fn = __vgic_v3_write_apxr1; 1096 break; 1097 case SYS_ICC_AP0Rn_EL1(2): 1098 case SYS_ICC_AP1Rn_EL1(2): 1099 if (is_read) 1100 fn = __vgic_v3_read_apxr2; 1101 else 1102 fn = __vgic_v3_write_apxr2; 1103 break; 1104 case SYS_ICC_AP0Rn_EL1(3): 1105 case SYS_ICC_AP1Rn_EL1(3): 1106 if (is_read) 1107 fn = __vgic_v3_read_apxr3; 1108 else 1109 fn = __vgic_v3_write_apxr3; 1110 break; 1111 case SYS_ICC_HPPIR0_EL1: 1112 case SYS_ICC_HPPIR1_EL1: 1113 if (unlikely(!is_read)) 1114 return 0; 1115 fn = __vgic_v3_read_hppir; 1116 break; 1117 case SYS_ICC_IGRPEN0_EL1: 1118 if (is_read) 1119 fn = __vgic_v3_read_igrpen0; 1120 else 1121 fn = __vgic_v3_write_igrpen0; 1122 break; 1123 case SYS_ICC_BPR0_EL1: 1124 if (is_read) 1125 fn = __vgic_v3_read_bpr0; 1126 else 1127 fn = __vgic_v3_write_bpr0; 1128 break; 1129 case SYS_ICC_DIR_EL1: 1130 if (unlikely(is_read)) 1131 return 0; 1132 fn = __vgic_v3_write_dir; 1133 break; 1134 case SYS_ICC_RPR_EL1: 1135 if (unlikely(!is_read)) 1136 return 0; 1137 fn = __vgic_v3_read_rpr; 1138 break; 1139 case SYS_ICC_CTLR_EL1: 1140 if (is_read) 1141 fn = __vgic_v3_read_ctlr; 1142 else 1143 fn = __vgic_v3_write_ctlr; 1144 break; 1145 case SYS_ICC_PMR_EL1: 1146 if (is_read) 1147 fn = __vgic_v3_read_pmr; 1148 else 1149 fn = __vgic_v3_write_pmr; 1150 break; 1151 default: 1152 return 0; 1153 } 1154 1155 vmcr = __vgic_v3_read_vmcr(); 1156 rt = kvm_vcpu_sys_get_rt(vcpu); 1157 fn(vcpu, vmcr, rt); 1158 1159 __kvm_skip_instr(vcpu); 1160 1161 return 1; 1162 } 1163