1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012-2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <hyp/adjust_pc.h>
8
9 #include <linux/compiler.h>
10 #include <linux/irqchip/arm-gic-v3.h>
11 #include <linux/kvm_host.h>
12
13 #include <asm/kvm_emulate.h>
14 #include <asm/kvm_hyp.h>
15 #include <asm/kvm_mmu.h>
16
17 #define vtr_to_max_lr_idx(v) ((v) & 0xf)
18 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1)
19 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5))
20
__gic_v3_get_lr(unsigned int lr)21 u64 __gic_v3_get_lr(unsigned int lr)
22 {
23 switch (lr & 0xf) {
24 case 0:
25 return read_gicreg(ICH_LR0_EL2);
26 case 1:
27 return read_gicreg(ICH_LR1_EL2);
28 case 2:
29 return read_gicreg(ICH_LR2_EL2);
30 case 3:
31 return read_gicreg(ICH_LR3_EL2);
32 case 4:
33 return read_gicreg(ICH_LR4_EL2);
34 case 5:
35 return read_gicreg(ICH_LR5_EL2);
36 case 6:
37 return read_gicreg(ICH_LR6_EL2);
38 case 7:
39 return read_gicreg(ICH_LR7_EL2);
40 case 8:
41 return read_gicreg(ICH_LR8_EL2);
42 case 9:
43 return read_gicreg(ICH_LR9_EL2);
44 case 10:
45 return read_gicreg(ICH_LR10_EL2);
46 case 11:
47 return read_gicreg(ICH_LR11_EL2);
48 case 12:
49 return read_gicreg(ICH_LR12_EL2);
50 case 13:
51 return read_gicreg(ICH_LR13_EL2);
52 case 14:
53 return read_gicreg(ICH_LR14_EL2);
54 case 15:
55 return read_gicreg(ICH_LR15_EL2);
56 }
57
58 unreachable();
59 }
60
__gic_v3_set_lr(u64 val,int lr)61 static void __gic_v3_set_lr(u64 val, int lr)
62 {
63 switch (lr & 0xf) {
64 case 0:
65 write_gicreg(val, ICH_LR0_EL2);
66 break;
67 case 1:
68 write_gicreg(val, ICH_LR1_EL2);
69 break;
70 case 2:
71 write_gicreg(val, ICH_LR2_EL2);
72 break;
73 case 3:
74 write_gicreg(val, ICH_LR3_EL2);
75 break;
76 case 4:
77 write_gicreg(val, ICH_LR4_EL2);
78 break;
79 case 5:
80 write_gicreg(val, ICH_LR5_EL2);
81 break;
82 case 6:
83 write_gicreg(val, ICH_LR6_EL2);
84 break;
85 case 7:
86 write_gicreg(val, ICH_LR7_EL2);
87 break;
88 case 8:
89 write_gicreg(val, ICH_LR8_EL2);
90 break;
91 case 9:
92 write_gicreg(val, ICH_LR9_EL2);
93 break;
94 case 10:
95 write_gicreg(val, ICH_LR10_EL2);
96 break;
97 case 11:
98 write_gicreg(val, ICH_LR11_EL2);
99 break;
100 case 12:
101 write_gicreg(val, ICH_LR12_EL2);
102 break;
103 case 13:
104 write_gicreg(val, ICH_LR13_EL2);
105 break;
106 case 14:
107 write_gicreg(val, ICH_LR14_EL2);
108 break;
109 case 15:
110 write_gicreg(val, ICH_LR15_EL2);
111 break;
112 }
113 }
114
__vgic_v3_write_ap0rn(u32 val,int n)115 static void __vgic_v3_write_ap0rn(u32 val, int n)
116 {
117 switch (n) {
118 case 0:
119 write_gicreg(val, ICH_AP0R0_EL2);
120 break;
121 case 1:
122 write_gicreg(val, ICH_AP0R1_EL2);
123 break;
124 case 2:
125 write_gicreg(val, ICH_AP0R2_EL2);
126 break;
127 case 3:
128 write_gicreg(val, ICH_AP0R3_EL2);
129 break;
130 }
131 }
132
__vgic_v3_write_ap1rn(u32 val,int n)133 static void __vgic_v3_write_ap1rn(u32 val, int n)
134 {
135 switch (n) {
136 case 0:
137 write_gicreg(val, ICH_AP1R0_EL2);
138 break;
139 case 1:
140 write_gicreg(val, ICH_AP1R1_EL2);
141 break;
142 case 2:
143 write_gicreg(val, ICH_AP1R2_EL2);
144 break;
145 case 3:
146 write_gicreg(val, ICH_AP1R3_EL2);
147 break;
148 }
149 }
150
__vgic_v3_read_ap0rn(int n)151 static u32 __vgic_v3_read_ap0rn(int n)
152 {
153 u32 val;
154
155 switch (n) {
156 case 0:
157 val = read_gicreg(ICH_AP0R0_EL2);
158 break;
159 case 1:
160 val = read_gicreg(ICH_AP0R1_EL2);
161 break;
162 case 2:
163 val = read_gicreg(ICH_AP0R2_EL2);
164 break;
165 case 3:
166 val = read_gicreg(ICH_AP0R3_EL2);
167 break;
168 default:
169 unreachable();
170 }
171
172 return val;
173 }
174
__vgic_v3_read_ap1rn(int n)175 static u32 __vgic_v3_read_ap1rn(int n)
176 {
177 u32 val;
178
179 switch (n) {
180 case 0:
181 val = read_gicreg(ICH_AP1R0_EL2);
182 break;
183 case 1:
184 val = read_gicreg(ICH_AP1R1_EL2);
185 break;
186 case 2:
187 val = read_gicreg(ICH_AP1R2_EL2);
188 break;
189 case 3:
190 val = read_gicreg(ICH_AP1R3_EL2);
191 break;
192 default:
193 unreachable();
194 }
195
196 return val;
197 }
198
__vgic_v3_save_state(struct vgic_v3_cpu_if * cpu_if)199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
200 {
201 u64 used_lrs = cpu_if->used_lrs;
202
203 /*
204 * Make sure stores to the GIC via the memory mapped interface
205 * are now visible to the system register interface when reading the
206 * LRs, and when reading back the VMCR on non-VHE systems.
207 */
208 if (used_lrs || !has_vhe()) {
209 if (!cpu_if->vgic_sre) {
210 dsb(sy);
211 isb();
212 }
213 }
214
215 if (used_lrs || cpu_if->its_vpe.its_vm) {
216 int i;
217 u32 elrsr;
218
219 elrsr = read_gicreg(ICH_ELRSR_EL2);
220
221 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EL2_En, ICH_HCR_EL2);
222
223 for (i = 0; i < used_lrs; i++) {
224 if (elrsr & (1 << i))
225 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
226 else
227 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
228
229 __gic_v3_set_lr(0, i);
230 }
231 }
232 }
233
__vgic_v3_restore_state(struct vgic_v3_cpu_if * cpu_if)234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
235 {
236 u64 used_lrs = cpu_if->used_lrs;
237 int i;
238
239 if (used_lrs || cpu_if->its_vpe.its_vm) {
240 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
241
242 for (i = 0; i < used_lrs; i++)
243 __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
244 }
245
246 /*
247 * Ensure that writes to the LRs, and on non-VHE systems ensure that
248 * the write to the VMCR in __vgic_v3_activate_traps(), will have
249 * reached the (re)distributors. This ensure the guest will read the
250 * correct values from the memory-mapped interface.
251 */
252 if (used_lrs || !has_vhe()) {
253 if (!cpu_if->vgic_sre) {
254 isb();
255 dsb(sy);
256 }
257 }
258 }
259
__vgic_v3_activate_traps(struct vgic_v3_cpu_if * cpu_if)260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
261 {
262 /*
263 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
264 * Group0 interrupt (as generated in GICv2 mode) to be
265 * delivered as a FIQ to the guest, with potentially fatal
266 * consequences. So we must make sure that ICC_SRE_EL1 has
267 * been actually programmed with the value we want before
268 * starting to mess with the rest of the GIC, and VMCR_EL2 in
269 * particular. This logic must be called before
270 * __vgic_v3_restore_state().
271 *
272 * However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is
273 * provisioned at all. In order to prevent illegal accesses to the
274 * system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1
275 * so that the trap bits can take effect. Yes, we *loves* the GIC.
276 */
277 if (!(cpu_if->vgic_hcr & ICH_HCR_EL2_En)) {
278 write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1);
279 isb();
280 } else if (!cpu_if->vgic_sre) {
281 write_gicreg(0, ICC_SRE_EL1);
282 isb();
283 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
284
285
286 if (has_vhe()) {
287 /*
288 * Ensure that the write to the VMCR will have reached
289 * the (re)distributors. This ensure the guest will
290 * read the correct values from the memory-mapped
291 * interface.
292 */
293 isb();
294 dsb(sy);
295 }
296 }
297
298 /*
299 * Prevent the guest from touching the ICC_SRE_EL1 system
300 * register. Note that this may not have any effect, as
301 * ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
302 */
303 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
304 ICC_SRE_EL2);
305
306 /*
307 * If we need to trap system registers, we must write
308 * ICH_HCR_EL2 anyway, even if no interrupts are being
309 * injected. Note that this also applies if we don't expect
310 * any system register access (no vgic at all).
311 */
312 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
313 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
314 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
315 }
316
__vgic_v3_deactivate_traps(struct vgic_v3_cpu_if * cpu_if)317 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
318 {
319 u64 val;
320
321 if (!cpu_if->vgic_sre) {
322 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
323 }
324
325 val = read_gicreg(ICC_SRE_EL2);
326 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
327
328 if (!cpu_if->vgic_sre) {
329 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
330 isb();
331 write_gicreg(1, ICC_SRE_EL1);
332 }
333
334 /*
335 * If we were trapping system registers, we enabled the VGIC even if
336 * no interrupts were being injected, and we disable it again here.
337 */
338 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
339 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
340 write_gicreg(0, ICH_HCR_EL2);
341 }
342
__vgic_v3_save_aprs(struct vgic_v3_cpu_if * cpu_if)343 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
344 {
345 u64 val;
346 u32 nr_pre_bits;
347
348 val = read_gicreg(ICH_VTR_EL2);
349 nr_pre_bits = vtr_to_nr_pre_bits(val);
350
351 switch (nr_pre_bits) {
352 case 7:
353 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
354 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
355 fallthrough;
356 case 6:
357 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
358 fallthrough;
359 default:
360 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
361 }
362
363 switch (nr_pre_bits) {
364 case 7:
365 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
366 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
367 fallthrough;
368 case 6:
369 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
370 fallthrough;
371 default:
372 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
373 }
374 }
375
__vgic_v3_restore_aprs(struct vgic_v3_cpu_if * cpu_if)376 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
377 {
378 u64 val;
379 u32 nr_pre_bits;
380
381 val = read_gicreg(ICH_VTR_EL2);
382 nr_pre_bits = vtr_to_nr_pre_bits(val);
383
384 switch (nr_pre_bits) {
385 case 7:
386 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
387 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
388 fallthrough;
389 case 6:
390 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
391 fallthrough;
392 default:
393 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
394 }
395
396 switch (nr_pre_bits) {
397 case 7:
398 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
399 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
400 fallthrough;
401 case 6:
402 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
403 fallthrough;
404 default:
405 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
406 }
407 }
408
__vgic_v3_init_lrs(void)409 void __vgic_v3_init_lrs(void)
410 {
411 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
412 int i;
413
414 for (i = 0; i <= max_lr_idx; i++)
415 __gic_v3_set_lr(0, i);
416 }
417
418 /*
419 * Return the GIC CPU configuration:
420 * - [31:0] ICH_VTR_EL2
421 * - [62:32] RES0
422 * - [63] MMIO (GICv2) capable
423 */
__vgic_v3_get_gic_config(void)424 u64 __vgic_v3_get_gic_config(void)
425 {
426 u64 val, sre = read_gicreg(ICC_SRE_EL1);
427 unsigned long flags = 0;
428
429 /*
430 * To check whether we have a MMIO-based (GICv2 compatible)
431 * CPU interface, we need to disable the system register
432 * view.
433 *
434 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
435 * that to be able to set ICC_SRE_EL1.SRE to 0, all the
436 * interrupt overrides must be set. You've got to love this.
437 *
438 * As we always run VHE with HCR_xMO set, no extra xMO
439 * manipulation is required in that case.
440 *
441 * To safely disable SRE, we have to prevent any interrupt
442 * from firing (which would be deadly). This only makes sense
443 * on VHE, as interrupts are already masked for nVHE as part
444 * of the exception entry to EL2.
445 */
446 if (has_vhe()) {
447 flags = local_daif_save();
448 } else {
449 sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO);
450 isb();
451 }
452
453 write_gicreg(0, ICC_SRE_EL1);
454 isb();
455
456 val = read_gicreg(ICC_SRE_EL1);
457
458 write_gicreg(sre, ICC_SRE_EL1);
459 isb();
460
461 if (has_vhe()) {
462 local_daif_restore(flags);
463 } else {
464 sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0);
465 isb();
466 }
467
468 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
469 val |= read_gicreg(ICH_VTR_EL2);
470
471 return val;
472 }
473
__vgic_v3_read_vmcr(void)474 static u64 __vgic_v3_read_vmcr(void)
475 {
476 return read_gicreg(ICH_VMCR_EL2);
477 }
478
__vgic_v3_write_vmcr(u32 vmcr)479 static void __vgic_v3_write_vmcr(u32 vmcr)
480 {
481 write_gicreg(vmcr, ICH_VMCR_EL2);
482 }
483
__vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if * cpu_if)484 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
485 {
486 __vgic_v3_save_aprs(cpu_if);
487 if (cpu_if->vgic_sre)
488 cpu_if->vgic_vmcr = __vgic_v3_read_vmcr();
489 }
490
__vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if * cpu_if)491 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
492 {
493 /*
494 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
495 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
496 * VMCR_EL2 save/restore in the world switch.
497 */
498 if (cpu_if->vgic_sre)
499 __vgic_v3_write_vmcr(cpu_if->vgic_vmcr);
500 __vgic_v3_restore_aprs(cpu_if);
501 }
502
__vgic_v3_bpr_min(void)503 static int __vgic_v3_bpr_min(void)
504 {
505 /* See Pseudocode for VPriorityGroup */
506 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
507 }
508
__vgic_v3_get_group(struct kvm_vcpu * vcpu)509 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
510 {
511 u64 esr = kvm_vcpu_get_esr(vcpu);
512 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
513
514 return crm != 8;
515 }
516
517 #define GICv3_IDLE_PRIORITY 0xff
518
__vgic_v3_highest_priority_lr(struct kvm_vcpu * vcpu,u32 vmcr,u64 * lr_val)519 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
520 u64 *lr_val)
521 {
522 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
523 u8 priority = GICv3_IDLE_PRIORITY;
524 int i, lr = -1;
525
526 for (i = 0; i < used_lrs; i++) {
527 u64 val = __gic_v3_get_lr(i);
528 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
529
530 /* Not pending in the state? */
531 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
532 continue;
533
534 /* Group-0 interrupt, but Group-0 disabled? */
535 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
536 continue;
537
538 /* Group-1 interrupt, but Group-1 disabled? */
539 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
540 continue;
541
542 /* Not the highest priority? */
543 if (lr_prio >= priority)
544 continue;
545
546 /* This is a candidate */
547 priority = lr_prio;
548 *lr_val = val;
549 lr = i;
550 }
551
552 if (lr == -1)
553 *lr_val = ICC_IAR1_EL1_SPURIOUS;
554
555 return lr;
556 }
557
__vgic_v3_find_active_lr(struct kvm_vcpu * vcpu,int intid,u64 * lr_val)558 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
559 u64 *lr_val)
560 {
561 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
562 int i;
563
564 for (i = 0; i < used_lrs; i++) {
565 u64 val = __gic_v3_get_lr(i);
566
567 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
568 (val & ICH_LR_ACTIVE_BIT)) {
569 *lr_val = val;
570 return i;
571 }
572 }
573
574 *lr_val = ICC_IAR1_EL1_SPURIOUS;
575 return -1;
576 }
577
__vgic_v3_get_highest_active_priority(void)578 static int __vgic_v3_get_highest_active_priority(void)
579 {
580 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
581 u32 hap = 0;
582 int i;
583
584 for (i = 0; i < nr_apr_regs; i++) {
585 u32 val;
586
587 /*
588 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
589 * contain the active priority levels for this VCPU
590 * for the maximum number of supported priority
591 * levels, and we return the full priority level only
592 * if the BPR is programmed to its minimum, otherwise
593 * we return a combination of the priority level and
594 * subpriority, as determined by the setting of the
595 * BPR, but without the full subpriority.
596 */
597 val = __vgic_v3_read_ap0rn(i);
598 val |= __vgic_v3_read_ap1rn(i);
599 if (!val) {
600 hap += 32;
601 continue;
602 }
603
604 return (hap + __ffs(val)) << __vgic_v3_bpr_min();
605 }
606
607 return GICv3_IDLE_PRIORITY;
608 }
609
__vgic_v3_get_bpr0(u32 vmcr)610 static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
611 {
612 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
613 }
614
__vgic_v3_get_bpr1(u32 vmcr)615 static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
616 {
617 unsigned int bpr;
618
619 if (vmcr & ICH_VMCR_CBPR_MASK) {
620 bpr = __vgic_v3_get_bpr0(vmcr);
621 if (bpr < 7)
622 bpr++;
623 } else {
624 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
625 }
626
627 return bpr;
628 }
629
630 /*
631 * Convert a priority to a preemption level, taking the relevant BPR
632 * into account by zeroing the sub-priority bits.
633 */
__vgic_v3_pri_to_pre(u8 pri,u32 vmcr,int grp)634 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
635 {
636 unsigned int bpr;
637
638 if (!grp)
639 bpr = __vgic_v3_get_bpr0(vmcr) + 1;
640 else
641 bpr = __vgic_v3_get_bpr1(vmcr);
642
643 return pri & (GENMASK(7, 0) << bpr);
644 }
645
646 /*
647 * The priority value is independent of any of the BPR values, so we
648 * normalize it using the minimal BPR value. This guarantees that no
649 * matter what the guest does with its BPR, we can always set/get the
650 * same value of a priority.
651 */
__vgic_v3_set_active_priority(u8 pri,u32 vmcr,int grp)652 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
653 {
654 u8 pre, ap;
655 u32 val;
656 int apr;
657
658 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
659 ap = pre >> __vgic_v3_bpr_min();
660 apr = ap / 32;
661
662 if (!grp) {
663 val = __vgic_v3_read_ap0rn(apr);
664 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
665 } else {
666 val = __vgic_v3_read_ap1rn(apr);
667 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
668 }
669 }
670
__vgic_v3_clear_highest_active_priority(void)671 static int __vgic_v3_clear_highest_active_priority(void)
672 {
673 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
674 u32 hap = 0;
675 int i;
676
677 for (i = 0; i < nr_apr_regs; i++) {
678 u32 ap0, ap1;
679 int c0, c1;
680
681 ap0 = __vgic_v3_read_ap0rn(i);
682 ap1 = __vgic_v3_read_ap1rn(i);
683 if (!ap0 && !ap1) {
684 hap += 32;
685 continue;
686 }
687
688 c0 = ap0 ? __ffs(ap0) : 32;
689 c1 = ap1 ? __ffs(ap1) : 32;
690
691 /* Always clear the LSB, which is the highest priority */
692 if (c0 < c1) {
693 ap0 &= ~BIT(c0);
694 __vgic_v3_write_ap0rn(ap0, i);
695 hap += c0;
696 } else {
697 ap1 &= ~BIT(c1);
698 __vgic_v3_write_ap1rn(ap1, i);
699 hap += c1;
700 }
701
702 /* Rescale to 8 bits of priority */
703 return hap << __vgic_v3_bpr_min();
704 }
705
706 return GICv3_IDLE_PRIORITY;
707 }
708
__vgic_v3_read_iar(struct kvm_vcpu * vcpu,u32 vmcr,int rt)709 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
710 {
711 u64 lr_val;
712 u8 lr_prio, pmr;
713 int lr, grp;
714
715 grp = __vgic_v3_get_group(vcpu);
716
717 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
718 if (lr < 0)
719 goto spurious;
720
721 if (grp != !!(lr_val & ICH_LR_GROUP))
722 goto spurious;
723
724 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
725 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
726 if (pmr <= lr_prio)
727 goto spurious;
728
729 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
730 goto spurious;
731
732 lr_val &= ~ICH_LR_STATE;
733 lr_val |= ICH_LR_ACTIVE_BIT;
734 __gic_v3_set_lr(lr_val, lr);
735 __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
736 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
737 return;
738
739 spurious:
740 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
741 }
742
__vgic_v3_clear_active_lr(int lr,u64 lr_val)743 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
744 {
745 lr_val &= ~ICH_LR_ACTIVE_BIT;
746 if (lr_val & ICH_LR_HW) {
747 u32 pid;
748
749 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
750 gic_write_dir(pid);
751 }
752
753 __gic_v3_set_lr(lr_val, lr);
754 }
755
__vgic_v3_bump_eoicount(void)756 static void __vgic_v3_bump_eoicount(void)
757 {
758 u32 hcr;
759
760 hcr = read_gicreg(ICH_HCR_EL2);
761 hcr += 1 << ICH_HCR_EL2_EOIcount_SHIFT;
762 write_gicreg(hcr, ICH_HCR_EL2);
763 }
764
__vgic_v3_write_dir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)765 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
766 {
767 u32 vid = vcpu_get_reg(vcpu, rt);
768 u64 lr_val;
769 int lr;
770
771 /* EOImode == 0, nothing to be done here */
772 if (!(vmcr & ICH_VMCR_EOIM_MASK))
773 return;
774
775 /* No deactivate to be performed on an LPI */
776 if (vid >= VGIC_MIN_LPI)
777 return;
778
779 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
780 if (lr == -1) {
781 __vgic_v3_bump_eoicount();
782 return;
783 }
784
785 __vgic_v3_clear_active_lr(lr, lr_val);
786 }
787
__vgic_v3_write_eoir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)788 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
789 {
790 u32 vid = vcpu_get_reg(vcpu, rt);
791 u64 lr_val;
792 u8 lr_prio, act_prio;
793 int lr, grp;
794
795 grp = __vgic_v3_get_group(vcpu);
796
797 /* Drop priority in any case */
798 act_prio = __vgic_v3_clear_highest_active_priority();
799
800 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
801 if (lr == -1) {
802 /* Do not bump EOIcount for LPIs that aren't in the LRs */
803 if (!(vid >= VGIC_MIN_LPI))
804 __vgic_v3_bump_eoicount();
805 return;
806 }
807
808 /* EOImode == 1 and not an LPI, nothing to be done here */
809 if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
810 return;
811
812 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
813
814 /* If priorities or group do not match, the guest has fscked-up. */
815 if (grp != !!(lr_val & ICH_LR_GROUP) ||
816 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
817 return;
818
819 /* Let's now perform the deactivation */
820 __vgic_v3_clear_active_lr(lr, lr_val);
821 }
822
__vgic_v3_read_igrpen0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)823 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
824 {
825 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
826 }
827
__vgic_v3_read_igrpen1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)828 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
829 {
830 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
831 }
832
__vgic_v3_write_igrpen0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)833 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
834 {
835 u64 val = vcpu_get_reg(vcpu, rt);
836
837 if (val & 1)
838 vmcr |= ICH_VMCR_ENG0_MASK;
839 else
840 vmcr &= ~ICH_VMCR_ENG0_MASK;
841
842 __vgic_v3_write_vmcr(vmcr);
843 }
844
__vgic_v3_write_igrpen1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)845 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
846 {
847 u64 val = vcpu_get_reg(vcpu, rt);
848
849 if (val & 1)
850 vmcr |= ICH_VMCR_ENG1_MASK;
851 else
852 vmcr &= ~ICH_VMCR_ENG1_MASK;
853
854 __vgic_v3_write_vmcr(vmcr);
855 }
856
__vgic_v3_read_bpr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)857 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
858 {
859 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
860 }
861
__vgic_v3_read_bpr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)862 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
863 {
864 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
865 }
866
__vgic_v3_write_bpr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)867 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
868 {
869 u64 val = vcpu_get_reg(vcpu, rt);
870 u8 bpr_min = __vgic_v3_bpr_min() - 1;
871
872 /* Enforce BPR limiting */
873 if (val < bpr_min)
874 val = bpr_min;
875
876 val <<= ICH_VMCR_BPR0_SHIFT;
877 val &= ICH_VMCR_BPR0_MASK;
878 vmcr &= ~ICH_VMCR_BPR0_MASK;
879 vmcr |= val;
880
881 __vgic_v3_write_vmcr(vmcr);
882 }
883
__vgic_v3_write_bpr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)884 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
885 {
886 u64 val = vcpu_get_reg(vcpu, rt);
887 u8 bpr_min = __vgic_v3_bpr_min();
888
889 if (vmcr & ICH_VMCR_CBPR_MASK)
890 return;
891
892 /* Enforce BPR limiting */
893 if (val < bpr_min)
894 val = bpr_min;
895
896 val <<= ICH_VMCR_BPR1_SHIFT;
897 val &= ICH_VMCR_BPR1_MASK;
898 vmcr &= ~ICH_VMCR_BPR1_MASK;
899 vmcr |= val;
900
901 __vgic_v3_write_vmcr(vmcr);
902 }
903
__vgic_v3_read_apxrn(struct kvm_vcpu * vcpu,int rt,int n)904 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
905 {
906 u32 val;
907
908 if (!__vgic_v3_get_group(vcpu))
909 val = __vgic_v3_read_ap0rn(n);
910 else
911 val = __vgic_v3_read_ap1rn(n);
912
913 vcpu_set_reg(vcpu, rt, val);
914 }
915
__vgic_v3_write_apxrn(struct kvm_vcpu * vcpu,int rt,int n)916 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
917 {
918 u32 val = vcpu_get_reg(vcpu, rt);
919
920 if (!__vgic_v3_get_group(vcpu))
921 __vgic_v3_write_ap0rn(val, n);
922 else
923 __vgic_v3_write_ap1rn(val, n);
924 }
925
__vgic_v3_read_apxr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)926 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
927 u32 vmcr, int rt)
928 {
929 __vgic_v3_read_apxrn(vcpu, rt, 0);
930 }
931
__vgic_v3_read_apxr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)932 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
933 u32 vmcr, int rt)
934 {
935 __vgic_v3_read_apxrn(vcpu, rt, 1);
936 }
937
__vgic_v3_read_apxr2(struct kvm_vcpu * vcpu,u32 vmcr,int rt)938 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
939 {
940 __vgic_v3_read_apxrn(vcpu, rt, 2);
941 }
942
__vgic_v3_read_apxr3(struct kvm_vcpu * vcpu,u32 vmcr,int rt)943 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
944 {
945 __vgic_v3_read_apxrn(vcpu, rt, 3);
946 }
947
__vgic_v3_write_apxr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)948 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
949 {
950 __vgic_v3_write_apxrn(vcpu, rt, 0);
951 }
952
__vgic_v3_write_apxr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)953 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
954 {
955 __vgic_v3_write_apxrn(vcpu, rt, 1);
956 }
957
__vgic_v3_write_apxr2(struct kvm_vcpu * vcpu,u32 vmcr,int rt)958 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
959 {
960 __vgic_v3_write_apxrn(vcpu, rt, 2);
961 }
962
__vgic_v3_write_apxr3(struct kvm_vcpu * vcpu,u32 vmcr,int rt)963 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
964 {
965 __vgic_v3_write_apxrn(vcpu, rt, 3);
966 }
967
__vgic_v3_read_hppir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)968 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
969 {
970 u64 lr_val;
971 int lr, lr_grp, grp;
972
973 grp = __vgic_v3_get_group(vcpu);
974
975 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
976 if (lr == -1)
977 goto spurious;
978
979 lr_grp = !!(lr_val & ICH_LR_GROUP);
980 if (lr_grp != grp)
981 lr_val = ICC_IAR1_EL1_SPURIOUS;
982
983 spurious:
984 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
985 }
986
__vgic_v3_read_pmr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)987 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
988 {
989 vmcr &= ICH_VMCR_PMR_MASK;
990 vmcr >>= ICH_VMCR_PMR_SHIFT;
991 vcpu_set_reg(vcpu, rt, vmcr);
992 }
993
__vgic_v3_write_pmr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)994 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
995 {
996 u32 val = vcpu_get_reg(vcpu, rt);
997
998 val <<= ICH_VMCR_PMR_SHIFT;
999 val &= ICH_VMCR_PMR_MASK;
1000 vmcr &= ~ICH_VMCR_PMR_MASK;
1001 vmcr |= val;
1002
1003 write_gicreg(vmcr, ICH_VMCR_EL2);
1004 }
1005
__vgic_v3_read_rpr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1006 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1007 {
1008 u32 val = __vgic_v3_get_highest_active_priority();
1009 vcpu_set_reg(vcpu, rt, val);
1010 }
1011
__vgic_v3_read_ctlr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1012 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1013 {
1014 u32 vtr, val;
1015
1016 vtr = read_gicreg(ICH_VTR_EL2);
1017 /* PRIbits */
1018 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
1019 /* IDbits */
1020 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
1021 /* A3V */
1022 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
1023 /* EOImode */
1024 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
1025 /* CBPR */
1026 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
1027
1028 vcpu_set_reg(vcpu, rt, val);
1029 }
1030
__vgic_v3_write_ctlr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1031 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1032 {
1033 u32 val = vcpu_get_reg(vcpu, rt);
1034
1035 if (val & ICC_CTLR_EL1_CBPR_MASK)
1036 vmcr |= ICH_VMCR_CBPR_MASK;
1037 else
1038 vmcr &= ~ICH_VMCR_CBPR_MASK;
1039
1040 if (val & ICC_CTLR_EL1_EOImode_MASK)
1041 vmcr |= ICH_VMCR_EOIM_MASK;
1042 else
1043 vmcr &= ~ICH_VMCR_EOIM_MASK;
1044
1045 write_gicreg(vmcr, ICH_VMCR_EL2);
1046 }
1047
__vgic_v3_check_trap_forwarding(struct kvm_vcpu * vcpu,u32 sysreg,bool is_read)1048 static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu,
1049 u32 sysreg, bool is_read)
1050 {
1051 u64 ich_hcr;
1052
1053 if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu))
1054 return false;
1055
1056 ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
1057
1058 switch (sysreg) {
1059 case SYS_ICC_IGRPEN0_EL1:
1060 if (is_read &&
1061 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
1062 return true;
1063
1064 if (!is_read &&
1065 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
1066 return true;
1067
1068 fallthrough;
1069
1070 case SYS_ICC_AP0Rn_EL1(0):
1071 case SYS_ICC_AP0Rn_EL1(1):
1072 case SYS_ICC_AP0Rn_EL1(2):
1073 case SYS_ICC_AP0Rn_EL1(3):
1074 case SYS_ICC_BPR0_EL1:
1075 case SYS_ICC_EOIR0_EL1:
1076 case SYS_ICC_HPPIR0_EL1:
1077 case SYS_ICC_IAR0_EL1:
1078 return ich_hcr & ICH_HCR_EL2_TALL0;
1079
1080 case SYS_ICC_IGRPEN1_EL1:
1081 if (is_read &&
1082 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
1083 return true;
1084
1085 if (!is_read &&
1086 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
1087 return true;
1088
1089 fallthrough;
1090
1091 case SYS_ICC_AP1Rn_EL1(0):
1092 case SYS_ICC_AP1Rn_EL1(1):
1093 case SYS_ICC_AP1Rn_EL1(2):
1094 case SYS_ICC_AP1Rn_EL1(3):
1095 case SYS_ICC_BPR1_EL1:
1096 case SYS_ICC_EOIR1_EL1:
1097 case SYS_ICC_HPPIR1_EL1:
1098 case SYS_ICC_IAR1_EL1:
1099 return ich_hcr & ICH_HCR_EL2_TALL1;
1100
1101 case SYS_ICC_DIR_EL1:
1102 if (ich_hcr & ICH_HCR_EL2_TDIR)
1103 return true;
1104
1105 fallthrough;
1106
1107 case SYS_ICC_RPR_EL1:
1108 case SYS_ICC_CTLR_EL1:
1109 case SYS_ICC_PMR_EL1:
1110 return ich_hcr & ICH_HCR_EL2_TC;
1111
1112 default:
1113 return false;
1114 }
1115 }
1116
__vgic_v3_perform_cpuif_access(struct kvm_vcpu * vcpu)1117 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1118 {
1119 int rt;
1120 u64 esr;
1121 u32 vmcr;
1122 void (*fn)(struct kvm_vcpu *, u32, int);
1123 bool is_read;
1124 u32 sysreg;
1125
1126 if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
1127 return 0;
1128
1129 esr = kvm_vcpu_get_esr(vcpu);
1130 if (vcpu_mode_is_32bit(vcpu)) {
1131 if (!kvm_condition_valid(vcpu)) {
1132 __kvm_skip_instr(vcpu);
1133 return 1;
1134 }
1135
1136 sysreg = esr_cp15_to_sysreg(esr);
1137 } else {
1138 sysreg = esr_sys64_to_sysreg(esr);
1139 }
1140
1141 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1142
1143 if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read))
1144 return 0;
1145
1146 switch (sysreg) {
1147 case SYS_ICC_IAR0_EL1:
1148 case SYS_ICC_IAR1_EL1:
1149 if (unlikely(!is_read))
1150 return 0;
1151 fn = __vgic_v3_read_iar;
1152 break;
1153 case SYS_ICC_EOIR0_EL1:
1154 case SYS_ICC_EOIR1_EL1:
1155 if (unlikely(is_read))
1156 return 0;
1157 fn = __vgic_v3_write_eoir;
1158 break;
1159 case SYS_ICC_IGRPEN1_EL1:
1160 if (is_read)
1161 fn = __vgic_v3_read_igrpen1;
1162 else
1163 fn = __vgic_v3_write_igrpen1;
1164 break;
1165 case SYS_ICC_BPR1_EL1:
1166 if (is_read)
1167 fn = __vgic_v3_read_bpr1;
1168 else
1169 fn = __vgic_v3_write_bpr1;
1170 break;
1171 case SYS_ICC_AP0Rn_EL1(0):
1172 case SYS_ICC_AP1Rn_EL1(0):
1173 if (is_read)
1174 fn = __vgic_v3_read_apxr0;
1175 else
1176 fn = __vgic_v3_write_apxr0;
1177 break;
1178 case SYS_ICC_AP0Rn_EL1(1):
1179 case SYS_ICC_AP1Rn_EL1(1):
1180 if (is_read)
1181 fn = __vgic_v3_read_apxr1;
1182 else
1183 fn = __vgic_v3_write_apxr1;
1184 break;
1185 case SYS_ICC_AP0Rn_EL1(2):
1186 case SYS_ICC_AP1Rn_EL1(2):
1187 if (is_read)
1188 fn = __vgic_v3_read_apxr2;
1189 else
1190 fn = __vgic_v3_write_apxr2;
1191 break;
1192 case SYS_ICC_AP0Rn_EL1(3):
1193 case SYS_ICC_AP1Rn_EL1(3):
1194 if (is_read)
1195 fn = __vgic_v3_read_apxr3;
1196 else
1197 fn = __vgic_v3_write_apxr3;
1198 break;
1199 case SYS_ICC_HPPIR0_EL1:
1200 case SYS_ICC_HPPIR1_EL1:
1201 if (unlikely(!is_read))
1202 return 0;
1203 fn = __vgic_v3_read_hppir;
1204 break;
1205 case SYS_ICC_IGRPEN0_EL1:
1206 if (is_read)
1207 fn = __vgic_v3_read_igrpen0;
1208 else
1209 fn = __vgic_v3_write_igrpen0;
1210 break;
1211 case SYS_ICC_BPR0_EL1:
1212 if (is_read)
1213 fn = __vgic_v3_read_bpr0;
1214 else
1215 fn = __vgic_v3_write_bpr0;
1216 break;
1217 case SYS_ICC_DIR_EL1:
1218 if (unlikely(is_read))
1219 return 0;
1220 fn = __vgic_v3_write_dir;
1221 break;
1222 case SYS_ICC_RPR_EL1:
1223 if (unlikely(!is_read))
1224 return 0;
1225 fn = __vgic_v3_read_rpr;
1226 break;
1227 case SYS_ICC_CTLR_EL1:
1228 if (is_read)
1229 fn = __vgic_v3_read_ctlr;
1230 else
1231 fn = __vgic_v3_write_ctlr;
1232 break;
1233 case SYS_ICC_PMR_EL1:
1234 if (is_read)
1235 fn = __vgic_v3_read_pmr;
1236 else
1237 fn = __vgic_v3_write_pmr;
1238 break;
1239 default:
1240 return 0;
1241 }
1242
1243 vmcr = __vgic_v3_read_vmcr();
1244 rt = kvm_vcpu_sys_get_rt(vcpu);
1245 fn(vcpu, vmcr, rt);
1246
1247 __kvm_skip_instr(vcpu);
1248
1249 return 1;
1250 }
1251