1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2012-2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7 #include <hyp/adjust_pc.h>
8
9 #include <linux/compiler.h>
10 #include <linux/irqchip/arm-gic-v3.h>
11 #include <linux/kvm_host.h>
12
13 #include <asm/kvm_emulate.h>
14 #include <asm/kvm_hyp.h>
15 #include <asm/kvm_mmu.h>
16
17 #define vtr_to_max_lr_idx(v) ((v) & 0xf)
18 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1)
19 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5))
20
__gic_v3_get_lr(unsigned int lr)21 u64 __gic_v3_get_lr(unsigned int lr)
22 {
23 switch (lr & 0xf) {
24 case 0:
25 return read_gicreg(ICH_LR0_EL2);
26 case 1:
27 return read_gicreg(ICH_LR1_EL2);
28 case 2:
29 return read_gicreg(ICH_LR2_EL2);
30 case 3:
31 return read_gicreg(ICH_LR3_EL2);
32 case 4:
33 return read_gicreg(ICH_LR4_EL2);
34 case 5:
35 return read_gicreg(ICH_LR5_EL2);
36 case 6:
37 return read_gicreg(ICH_LR6_EL2);
38 case 7:
39 return read_gicreg(ICH_LR7_EL2);
40 case 8:
41 return read_gicreg(ICH_LR8_EL2);
42 case 9:
43 return read_gicreg(ICH_LR9_EL2);
44 case 10:
45 return read_gicreg(ICH_LR10_EL2);
46 case 11:
47 return read_gicreg(ICH_LR11_EL2);
48 case 12:
49 return read_gicreg(ICH_LR12_EL2);
50 case 13:
51 return read_gicreg(ICH_LR13_EL2);
52 case 14:
53 return read_gicreg(ICH_LR14_EL2);
54 case 15:
55 return read_gicreg(ICH_LR15_EL2);
56 }
57
58 unreachable();
59 }
60
__gic_v3_set_lr(u64 val,int lr)61 static void __gic_v3_set_lr(u64 val, int lr)
62 {
63 switch (lr & 0xf) {
64 case 0:
65 write_gicreg(val, ICH_LR0_EL2);
66 break;
67 case 1:
68 write_gicreg(val, ICH_LR1_EL2);
69 break;
70 case 2:
71 write_gicreg(val, ICH_LR2_EL2);
72 break;
73 case 3:
74 write_gicreg(val, ICH_LR3_EL2);
75 break;
76 case 4:
77 write_gicreg(val, ICH_LR4_EL2);
78 break;
79 case 5:
80 write_gicreg(val, ICH_LR5_EL2);
81 break;
82 case 6:
83 write_gicreg(val, ICH_LR6_EL2);
84 break;
85 case 7:
86 write_gicreg(val, ICH_LR7_EL2);
87 break;
88 case 8:
89 write_gicreg(val, ICH_LR8_EL2);
90 break;
91 case 9:
92 write_gicreg(val, ICH_LR9_EL2);
93 break;
94 case 10:
95 write_gicreg(val, ICH_LR10_EL2);
96 break;
97 case 11:
98 write_gicreg(val, ICH_LR11_EL2);
99 break;
100 case 12:
101 write_gicreg(val, ICH_LR12_EL2);
102 break;
103 case 13:
104 write_gicreg(val, ICH_LR13_EL2);
105 break;
106 case 14:
107 write_gicreg(val, ICH_LR14_EL2);
108 break;
109 case 15:
110 write_gicreg(val, ICH_LR15_EL2);
111 break;
112 }
113 }
114
__vgic_v3_write_ap0rn(u32 val,int n)115 static void __vgic_v3_write_ap0rn(u32 val, int n)
116 {
117 switch (n) {
118 case 0:
119 write_gicreg(val, ICH_AP0R0_EL2);
120 break;
121 case 1:
122 write_gicreg(val, ICH_AP0R1_EL2);
123 break;
124 case 2:
125 write_gicreg(val, ICH_AP0R2_EL2);
126 break;
127 case 3:
128 write_gicreg(val, ICH_AP0R3_EL2);
129 break;
130 }
131 }
132
__vgic_v3_write_ap1rn(u32 val,int n)133 static void __vgic_v3_write_ap1rn(u32 val, int n)
134 {
135 switch (n) {
136 case 0:
137 write_gicreg(val, ICH_AP1R0_EL2);
138 break;
139 case 1:
140 write_gicreg(val, ICH_AP1R1_EL2);
141 break;
142 case 2:
143 write_gicreg(val, ICH_AP1R2_EL2);
144 break;
145 case 3:
146 write_gicreg(val, ICH_AP1R3_EL2);
147 break;
148 }
149 }
150
__vgic_v3_read_ap0rn(int n)151 static u32 __vgic_v3_read_ap0rn(int n)
152 {
153 u32 val;
154
155 switch (n) {
156 case 0:
157 val = read_gicreg(ICH_AP0R0_EL2);
158 break;
159 case 1:
160 val = read_gicreg(ICH_AP0R1_EL2);
161 break;
162 case 2:
163 val = read_gicreg(ICH_AP0R2_EL2);
164 break;
165 case 3:
166 val = read_gicreg(ICH_AP0R3_EL2);
167 break;
168 default:
169 unreachable();
170 }
171
172 return val;
173 }
174
__vgic_v3_read_ap1rn(int n)175 static u32 __vgic_v3_read_ap1rn(int n)
176 {
177 u32 val;
178
179 switch (n) {
180 case 0:
181 val = read_gicreg(ICH_AP1R0_EL2);
182 break;
183 case 1:
184 val = read_gicreg(ICH_AP1R1_EL2);
185 break;
186 case 2:
187 val = read_gicreg(ICH_AP1R2_EL2);
188 break;
189 case 3:
190 val = read_gicreg(ICH_AP1R3_EL2);
191 break;
192 default:
193 unreachable();
194 }
195
196 return val;
197 }
198
__vgic_v3_save_state(struct vgic_v3_cpu_if * cpu_if)199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
200 {
201 u64 used_lrs = cpu_if->used_lrs;
202
203 /*
204 * Make sure stores to the GIC via the memory mapped interface
205 * are now visible to the system register interface when reading the
206 * LRs, and when reading back the VMCR on non-VHE systems.
207 */
208 if (used_lrs || !has_vhe()) {
209 if (!cpu_if->vgic_sre) {
210 dsb(sy);
211 isb();
212 }
213 }
214
215 if (used_lrs || cpu_if->its_vpe.its_vm) {
216 int i;
217 u32 elrsr;
218
219 elrsr = read_gicreg(ICH_ELRSR_EL2);
220
221 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EL2_En, ICH_HCR_EL2);
222
223 for (i = 0; i < used_lrs; i++) {
224 if (elrsr & (1 << i))
225 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
226 else
227 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
228
229 __gic_v3_set_lr(0, i);
230 }
231 }
232 }
233
__vgic_v3_restore_state(struct vgic_v3_cpu_if * cpu_if)234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
235 {
236 u64 used_lrs = cpu_if->used_lrs;
237 int i;
238
239 if (used_lrs || cpu_if->its_vpe.its_vm) {
240 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
241
242 for (i = 0; i < used_lrs; i++)
243 __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
244 }
245
246 /*
247 * Ensure that writes to the LRs, and on non-VHE systems ensure that
248 * the write to the VMCR in __vgic_v3_activate_traps(), will have
249 * reached the (re)distributors. This ensure the guest will read the
250 * correct values from the memory-mapped interface.
251 */
252 if (used_lrs || !has_vhe()) {
253 if (!cpu_if->vgic_sre) {
254 isb();
255 dsb(sy);
256 }
257 }
258 }
259
__vgic_v3_activate_traps(struct vgic_v3_cpu_if * cpu_if)260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
261 {
262 /*
263 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
264 * Group0 interrupt (as generated in GICv2 mode) to be
265 * delivered as a FIQ to the guest, with potentially fatal
266 * consequences. So we must make sure that ICC_SRE_EL1 has
267 * been actually programmed with the value we want before
268 * starting to mess with the rest of the GIC, and VMCR_EL2 in
269 * particular. This logic must be called before
270 * __vgic_v3_restore_state().
271 *
272 * However, if the vgic is disabled (ICH_HCR_EL2.EN==0), no GIC is
273 * provisioned at all. In order to prevent illegal accesses to the
274 * system registers to trap to EL1 (duh), force ICC_SRE_EL1.SRE to 1
275 * so that the trap bits can take effect. Yes, we *loves* the GIC.
276 */
277 if (!(cpu_if->vgic_hcr & ICH_HCR_EL2_En)) {
278 write_gicreg(ICC_SRE_EL1_SRE, ICC_SRE_EL1);
279 isb();
280 } else if (!cpu_if->vgic_sre) {
281 write_gicreg(0, ICC_SRE_EL1);
282 isb();
283 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
284
285
286 if (has_vhe()) {
287 /*
288 * Ensure that the write to the VMCR will have reached
289 * the (re)distributors. This ensure the guest will
290 * read the correct values from the memory-mapped
291 * interface.
292 */
293 isb();
294 dsb(sy);
295 }
296 }
297
298 /*
299 * GICv5 BET0 FEAT_GCIE_LEGACY doesn't include ICC_SRE_EL2. This is due
300 * to be relaxed in a future spec release, at which point this in
301 * condition can be dropped.
302 */
303 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
304 /*
305 * Prevent the guest from touching the ICC_SRE_EL1 system
306 * register. Note that this may not have any effect, as
307 * ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
308 */
309 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
310 ICC_SRE_EL2);
311 }
312
313 /*
314 * If we need to trap system registers, we must write
315 * ICH_HCR_EL2 anyway, even if no interrupts are being
316 * injected. Note that this also applies if we don't expect
317 * any system register access (no vgic at all).
318 */
319 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
320 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
321 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
322 }
323
__vgic_v3_deactivate_traps(struct vgic_v3_cpu_if * cpu_if)324 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
325 {
326 u64 val;
327
328 if (!cpu_if->vgic_sre) {
329 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
330 }
331
332 /*
333 * Can be dropped in the future when GICv5 spec is relaxed. See comment
334 * above.
335 */
336 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
337 val = read_gicreg(ICC_SRE_EL2);
338 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
339 }
340
341 if (!cpu_if->vgic_sre) {
342 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
343 isb();
344 write_gicreg(1, ICC_SRE_EL1);
345 }
346
347 /*
348 * If we were trapping system registers, we enabled the VGIC even if
349 * no interrupts were being injected, and we disable it again here.
350 */
351 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
352 cpu_if->its_vpe.its_vm || !cpu_if->vgic_sre)
353 write_gicreg(0, ICH_HCR_EL2);
354 }
355
__vgic_v3_save_aprs(struct vgic_v3_cpu_if * cpu_if)356 static void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
357 {
358 u64 val;
359 u32 nr_pre_bits;
360
361 val = read_gicreg(ICH_VTR_EL2);
362 nr_pre_bits = vtr_to_nr_pre_bits(val);
363
364 switch (nr_pre_bits) {
365 case 7:
366 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
367 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
368 fallthrough;
369 case 6:
370 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
371 fallthrough;
372 default:
373 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
374 }
375
376 switch (nr_pre_bits) {
377 case 7:
378 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
379 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
380 fallthrough;
381 case 6:
382 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
383 fallthrough;
384 default:
385 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
386 }
387 }
388
__vgic_v3_restore_aprs(struct vgic_v3_cpu_if * cpu_if)389 static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
390 {
391 u64 val;
392 u32 nr_pre_bits;
393
394 val = read_gicreg(ICH_VTR_EL2);
395 nr_pre_bits = vtr_to_nr_pre_bits(val);
396
397 switch (nr_pre_bits) {
398 case 7:
399 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
400 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
401 fallthrough;
402 case 6:
403 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
404 fallthrough;
405 default:
406 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
407 }
408
409 switch (nr_pre_bits) {
410 case 7:
411 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
412 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
413 fallthrough;
414 case 6:
415 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
416 fallthrough;
417 default:
418 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
419 }
420 }
421
__vgic_v3_init_lrs(void)422 void __vgic_v3_init_lrs(void)
423 {
424 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
425 int i;
426
427 for (i = 0; i <= max_lr_idx; i++)
428 __gic_v3_set_lr(0, i);
429 }
430
431 /*
432 * Return the GIC CPU configuration:
433 * - [31:0] ICH_VTR_EL2
434 * - [62:32] RES0
435 * - [63] MMIO (GICv2) capable
436 */
__vgic_v3_get_gic_config(void)437 u64 __vgic_v3_get_gic_config(void)
438 {
439 u64 val, sre;
440 unsigned long flags = 0;
441
442 /*
443 * In compat mode, we cannot access ICC_SRE_EL1 at any EL
444 * other than EL1 itself; just return the
445 * ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5
446 * system, so we first check if we have GICv5 support.
447 */
448 if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
449 return read_gicreg(ICH_VTR_EL2);
450
451 sre = read_gicreg(ICC_SRE_EL1);
452 /*
453 * To check whether we have a MMIO-based (GICv2 compatible)
454 * CPU interface, we need to disable the system register
455 * view.
456 *
457 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
458 * that to be able to set ICC_SRE_EL1.SRE to 0, all the
459 * interrupt overrides must be set. You've got to love this.
460 *
461 * As we always run VHE with HCR_xMO set, no extra xMO
462 * manipulation is required in that case.
463 *
464 * To safely disable SRE, we have to prevent any interrupt
465 * from firing (which would be deadly). This only makes sense
466 * on VHE, as interrupts are already masked for nVHE as part
467 * of the exception entry to EL2.
468 */
469 if (has_vhe()) {
470 flags = local_daif_save();
471 } else {
472 sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO);
473 isb();
474 }
475
476 write_gicreg(0, ICC_SRE_EL1);
477 isb();
478
479 val = read_gicreg(ICC_SRE_EL1);
480
481 write_gicreg(sre, ICC_SRE_EL1);
482 isb();
483
484 if (has_vhe()) {
485 local_daif_restore(flags);
486 } else {
487 sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0);
488 isb();
489 }
490
491 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
492 val |= read_gicreg(ICH_VTR_EL2);
493
494 return val;
495 }
496
__vgic_v3_compat_mode_enable(void)497 static void __vgic_v3_compat_mode_enable(void)
498 {
499 if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
500 return;
501
502 sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3);
503 /* Wait for V3 to become enabled */
504 isb();
505 }
506
__vgic_v3_read_vmcr(void)507 static u64 __vgic_v3_read_vmcr(void)
508 {
509 return read_gicreg(ICH_VMCR_EL2);
510 }
511
__vgic_v3_write_vmcr(u32 vmcr)512 static void __vgic_v3_write_vmcr(u32 vmcr)
513 {
514 write_gicreg(vmcr, ICH_VMCR_EL2);
515 }
516
__vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if * cpu_if)517 void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
518 {
519 __vgic_v3_save_aprs(cpu_if);
520 if (cpu_if->vgic_sre)
521 cpu_if->vgic_vmcr = __vgic_v3_read_vmcr();
522 }
523
__vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if * cpu_if)524 void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
525 {
526 __vgic_v3_compat_mode_enable();
527
528 /*
529 * If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
530 * is dependent on ICC_SRE_EL1.SRE, and we have to perform the
531 * VMCR_EL2 save/restore in the world switch.
532 */
533 if (cpu_if->vgic_sre)
534 __vgic_v3_write_vmcr(cpu_if->vgic_vmcr);
535 __vgic_v3_restore_aprs(cpu_if);
536 }
537
__vgic_v3_bpr_min(void)538 static int __vgic_v3_bpr_min(void)
539 {
540 /* See Pseudocode for VPriorityGroup */
541 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
542 }
543
__vgic_v3_get_group(struct kvm_vcpu * vcpu)544 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
545 {
546 u64 esr = kvm_vcpu_get_esr(vcpu);
547 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
548
549 return crm != 8;
550 }
551
552 #define GICv3_IDLE_PRIORITY 0xff
553
__vgic_v3_highest_priority_lr(struct kvm_vcpu * vcpu,u32 vmcr,u64 * lr_val)554 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
555 u64 *lr_val)
556 {
557 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
558 u8 priority = GICv3_IDLE_PRIORITY;
559 int i, lr = -1;
560
561 for (i = 0; i < used_lrs; i++) {
562 u64 val = __gic_v3_get_lr(i);
563 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
564
565 /* Not pending in the state? */
566 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
567 continue;
568
569 /* Group-0 interrupt, but Group-0 disabled? */
570 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
571 continue;
572
573 /* Group-1 interrupt, but Group-1 disabled? */
574 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
575 continue;
576
577 /* Not the highest priority? */
578 if (lr_prio >= priority)
579 continue;
580
581 /* This is a candidate */
582 priority = lr_prio;
583 *lr_val = val;
584 lr = i;
585 }
586
587 if (lr == -1)
588 *lr_val = ICC_IAR1_EL1_SPURIOUS;
589
590 return lr;
591 }
592
__vgic_v3_find_active_lr(struct kvm_vcpu * vcpu,int intid,u64 * lr_val)593 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
594 u64 *lr_val)
595 {
596 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
597 int i;
598
599 for (i = 0; i < used_lrs; i++) {
600 u64 val = __gic_v3_get_lr(i);
601
602 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
603 (val & ICH_LR_ACTIVE_BIT)) {
604 *lr_val = val;
605 return i;
606 }
607 }
608
609 *lr_val = ICC_IAR1_EL1_SPURIOUS;
610 return -1;
611 }
612
__vgic_v3_get_highest_active_priority(void)613 static int __vgic_v3_get_highest_active_priority(void)
614 {
615 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
616 u32 hap = 0;
617 int i;
618
619 for (i = 0; i < nr_apr_regs; i++) {
620 u32 val;
621
622 /*
623 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
624 * contain the active priority levels for this VCPU
625 * for the maximum number of supported priority
626 * levels, and we return the full priority level only
627 * if the BPR is programmed to its minimum, otherwise
628 * we return a combination of the priority level and
629 * subpriority, as determined by the setting of the
630 * BPR, but without the full subpriority.
631 */
632 val = __vgic_v3_read_ap0rn(i);
633 val |= __vgic_v3_read_ap1rn(i);
634 if (!val) {
635 hap += 32;
636 continue;
637 }
638
639 return (hap + __ffs(val)) << __vgic_v3_bpr_min();
640 }
641
642 return GICv3_IDLE_PRIORITY;
643 }
644
__vgic_v3_get_bpr0(u32 vmcr)645 static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
646 {
647 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
648 }
649
__vgic_v3_get_bpr1(u32 vmcr)650 static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
651 {
652 unsigned int bpr;
653
654 if (vmcr & ICH_VMCR_CBPR_MASK) {
655 bpr = __vgic_v3_get_bpr0(vmcr);
656 if (bpr < 7)
657 bpr++;
658 } else {
659 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
660 }
661
662 return bpr;
663 }
664
665 /*
666 * Convert a priority to a preemption level, taking the relevant BPR
667 * into account by zeroing the sub-priority bits.
668 */
__vgic_v3_pri_to_pre(u8 pri,u32 vmcr,int grp)669 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
670 {
671 unsigned int bpr;
672
673 if (!grp)
674 bpr = __vgic_v3_get_bpr0(vmcr) + 1;
675 else
676 bpr = __vgic_v3_get_bpr1(vmcr);
677
678 return pri & (GENMASK(7, 0) << bpr);
679 }
680
681 /*
682 * The priority value is independent of any of the BPR values, so we
683 * normalize it using the minimal BPR value. This guarantees that no
684 * matter what the guest does with its BPR, we can always set/get the
685 * same value of a priority.
686 */
__vgic_v3_set_active_priority(u8 pri,u32 vmcr,int grp)687 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
688 {
689 u8 pre, ap;
690 u32 val;
691 int apr;
692
693 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
694 ap = pre >> __vgic_v3_bpr_min();
695 apr = ap / 32;
696
697 if (!grp) {
698 val = __vgic_v3_read_ap0rn(apr);
699 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
700 } else {
701 val = __vgic_v3_read_ap1rn(apr);
702 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
703 }
704 }
705
__vgic_v3_clear_highest_active_priority(void)706 static int __vgic_v3_clear_highest_active_priority(void)
707 {
708 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
709 u32 hap = 0;
710 int i;
711
712 for (i = 0; i < nr_apr_regs; i++) {
713 u32 ap0, ap1;
714 int c0, c1;
715
716 ap0 = __vgic_v3_read_ap0rn(i);
717 ap1 = __vgic_v3_read_ap1rn(i);
718 if (!ap0 && !ap1) {
719 hap += 32;
720 continue;
721 }
722
723 c0 = ap0 ? __ffs(ap0) : 32;
724 c1 = ap1 ? __ffs(ap1) : 32;
725
726 /* Always clear the LSB, which is the highest priority */
727 if (c0 < c1) {
728 ap0 &= ~BIT(c0);
729 __vgic_v3_write_ap0rn(ap0, i);
730 hap += c0;
731 } else {
732 ap1 &= ~BIT(c1);
733 __vgic_v3_write_ap1rn(ap1, i);
734 hap += c1;
735 }
736
737 /* Rescale to 8 bits of priority */
738 return hap << __vgic_v3_bpr_min();
739 }
740
741 return GICv3_IDLE_PRIORITY;
742 }
743
__vgic_v3_read_iar(struct kvm_vcpu * vcpu,u32 vmcr,int rt)744 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
745 {
746 u64 lr_val;
747 u8 lr_prio, pmr;
748 int lr, grp;
749
750 grp = __vgic_v3_get_group(vcpu);
751
752 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
753 if (lr < 0)
754 goto spurious;
755
756 if (grp != !!(lr_val & ICH_LR_GROUP))
757 goto spurious;
758
759 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
760 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
761 if (pmr <= lr_prio)
762 goto spurious;
763
764 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
765 goto spurious;
766
767 lr_val &= ~ICH_LR_STATE;
768 lr_val |= ICH_LR_ACTIVE_BIT;
769 __gic_v3_set_lr(lr_val, lr);
770 __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
771 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
772 return;
773
774 spurious:
775 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
776 }
777
__vgic_v3_clear_active_lr(int lr,u64 lr_val)778 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
779 {
780 lr_val &= ~ICH_LR_ACTIVE_BIT;
781 if (lr_val & ICH_LR_HW) {
782 u32 pid;
783
784 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
785 gic_write_dir(pid);
786 }
787
788 __gic_v3_set_lr(lr_val, lr);
789 }
790
__vgic_v3_bump_eoicount(void)791 static void __vgic_v3_bump_eoicount(void)
792 {
793 u32 hcr;
794
795 hcr = read_gicreg(ICH_HCR_EL2);
796 hcr += 1 << ICH_HCR_EL2_EOIcount_SHIFT;
797 write_gicreg(hcr, ICH_HCR_EL2);
798 }
799
__vgic_v3_write_dir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)800 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
801 {
802 u32 vid = vcpu_get_reg(vcpu, rt);
803 u64 lr_val;
804 int lr;
805
806 /* EOImode == 0, nothing to be done here */
807 if (!(vmcr & ICH_VMCR_EOIM_MASK))
808 return;
809
810 /* No deactivate to be performed on an LPI */
811 if (vid >= VGIC_MIN_LPI)
812 return;
813
814 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
815 if (lr == -1) {
816 __vgic_v3_bump_eoicount();
817 return;
818 }
819
820 __vgic_v3_clear_active_lr(lr, lr_val);
821 }
822
__vgic_v3_write_eoir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)823 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
824 {
825 u32 vid = vcpu_get_reg(vcpu, rt);
826 u64 lr_val;
827 u8 lr_prio, act_prio;
828 int lr, grp;
829
830 grp = __vgic_v3_get_group(vcpu);
831
832 /* Drop priority in any case */
833 act_prio = __vgic_v3_clear_highest_active_priority();
834
835 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
836 if (lr == -1) {
837 /* Do not bump EOIcount for LPIs that aren't in the LRs */
838 if (!(vid >= VGIC_MIN_LPI))
839 __vgic_v3_bump_eoicount();
840 return;
841 }
842
843 /* EOImode == 1 and not an LPI, nothing to be done here */
844 if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
845 return;
846
847 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
848
849 /* If priorities or group do not match, the guest has fscked-up. */
850 if (grp != !!(lr_val & ICH_LR_GROUP) ||
851 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
852 return;
853
854 /* Let's now perform the deactivation */
855 __vgic_v3_clear_active_lr(lr, lr_val);
856 }
857
__vgic_v3_read_igrpen0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)858 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
859 {
860 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
861 }
862
__vgic_v3_read_igrpen1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)863 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
864 {
865 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
866 }
867
__vgic_v3_write_igrpen0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)868 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
869 {
870 u64 val = vcpu_get_reg(vcpu, rt);
871
872 if (val & 1)
873 vmcr |= ICH_VMCR_ENG0_MASK;
874 else
875 vmcr &= ~ICH_VMCR_ENG0_MASK;
876
877 __vgic_v3_write_vmcr(vmcr);
878 }
879
__vgic_v3_write_igrpen1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)880 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
881 {
882 u64 val = vcpu_get_reg(vcpu, rt);
883
884 if (val & 1)
885 vmcr |= ICH_VMCR_ENG1_MASK;
886 else
887 vmcr &= ~ICH_VMCR_ENG1_MASK;
888
889 __vgic_v3_write_vmcr(vmcr);
890 }
891
__vgic_v3_read_bpr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)892 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
893 {
894 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
895 }
896
__vgic_v3_read_bpr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)897 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
898 {
899 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
900 }
901
__vgic_v3_write_bpr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)902 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
903 {
904 u64 val = vcpu_get_reg(vcpu, rt);
905 u8 bpr_min = __vgic_v3_bpr_min() - 1;
906
907 /* Enforce BPR limiting */
908 if (val < bpr_min)
909 val = bpr_min;
910
911 val <<= ICH_VMCR_BPR0_SHIFT;
912 val &= ICH_VMCR_BPR0_MASK;
913 vmcr &= ~ICH_VMCR_BPR0_MASK;
914 vmcr |= val;
915
916 __vgic_v3_write_vmcr(vmcr);
917 }
918
__vgic_v3_write_bpr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)919 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
920 {
921 u64 val = vcpu_get_reg(vcpu, rt);
922 u8 bpr_min = __vgic_v3_bpr_min();
923
924 if (vmcr & ICH_VMCR_CBPR_MASK)
925 return;
926
927 /* Enforce BPR limiting */
928 if (val < bpr_min)
929 val = bpr_min;
930
931 val <<= ICH_VMCR_BPR1_SHIFT;
932 val &= ICH_VMCR_BPR1_MASK;
933 vmcr &= ~ICH_VMCR_BPR1_MASK;
934 vmcr |= val;
935
936 __vgic_v3_write_vmcr(vmcr);
937 }
938
__vgic_v3_read_apxrn(struct kvm_vcpu * vcpu,int rt,int n)939 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
940 {
941 u32 val;
942
943 if (!__vgic_v3_get_group(vcpu))
944 val = __vgic_v3_read_ap0rn(n);
945 else
946 val = __vgic_v3_read_ap1rn(n);
947
948 vcpu_set_reg(vcpu, rt, val);
949 }
950
__vgic_v3_write_apxrn(struct kvm_vcpu * vcpu,int rt,int n)951 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
952 {
953 u32 val = vcpu_get_reg(vcpu, rt);
954
955 if (!__vgic_v3_get_group(vcpu))
956 __vgic_v3_write_ap0rn(val, n);
957 else
958 __vgic_v3_write_ap1rn(val, n);
959 }
960
__vgic_v3_read_apxr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)961 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
962 u32 vmcr, int rt)
963 {
964 __vgic_v3_read_apxrn(vcpu, rt, 0);
965 }
966
__vgic_v3_read_apxr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)967 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
968 u32 vmcr, int rt)
969 {
970 __vgic_v3_read_apxrn(vcpu, rt, 1);
971 }
972
__vgic_v3_read_apxr2(struct kvm_vcpu * vcpu,u32 vmcr,int rt)973 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
974 {
975 __vgic_v3_read_apxrn(vcpu, rt, 2);
976 }
977
__vgic_v3_read_apxr3(struct kvm_vcpu * vcpu,u32 vmcr,int rt)978 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
979 {
980 __vgic_v3_read_apxrn(vcpu, rt, 3);
981 }
982
__vgic_v3_write_apxr0(struct kvm_vcpu * vcpu,u32 vmcr,int rt)983 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
984 {
985 __vgic_v3_write_apxrn(vcpu, rt, 0);
986 }
987
__vgic_v3_write_apxr1(struct kvm_vcpu * vcpu,u32 vmcr,int rt)988 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
989 {
990 __vgic_v3_write_apxrn(vcpu, rt, 1);
991 }
992
__vgic_v3_write_apxr2(struct kvm_vcpu * vcpu,u32 vmcr,int rt)993 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
994 {
995 __vgic_v3_write_apxrn(vcpu, rt, 2);
996 }
997
__vgic_v3_write_apxr3(struct kvm_vcpu * vcpu,u32 vmcr,int rt)998 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
999 {
1000 __vgic_v3_write_apxrn(vcpu, rt, 3);
1001 }
1002
__vgic_v3_read_hppir(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1003 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1004 {
1005 u64 lr_val;
1006 int lr, lr_grp, grp;
1007
1008 grp = __vgic_v3_get_group(vcpu);
1009
1010 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
1011 if (lr == -1)
1012 goto spurious;
1013
1014 lr_grp = !!(lr_val & ICH_LR_GROUP);
1015 if (lr_grp != grp)
1016 lr_val = ICC_IAR1_EL1_SPURIOUS;
1017
1018 spurious:
1019 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
1020 }
1021
__vgic_v3_read_pmr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1022 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1023 {
1024 vmcr &= ICH_VMCR_PMR_MASK;
1025 vmcr >>= ICH_VMCR_PMR_SHIFT;
1026 vcpu_set_reg(vcpu, rt, vmcr);
1027 }
1028
__vgic_v3_write_pmr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1029 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1030 {
1031 u32 val = vcpu_get_reg(vcpu, rt);
1032
1033 val <<= ICH_VMCR_PMR_SHIFT;
1034 val &= ICH_VMCR_PMR_MASK;
1035 vmcr &= ~ICH_VMCR_PMR_MASK;
1036 vmcr |= val;
1037
1038 write_gicreg(vmcr, ICH_VMCR_EL2);
1039 }
1040
__vgic_v3_read_rpr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1041 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1042 {
1043 u32 val = __vgic_v3_get_highest_active_priority();
1044 vcpu_set_reg(vcpu, rt, val);
1045 }
1046
__vgic_v3_read_ctlr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1047 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1048 {
1049 u32 vtr, val;
1050
1051 vtr = read_gicreg(ICH_VTR_EL2);
1052 /* PRIbits */
1053 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
1054 /* IDbits */
1055 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
1056 /* A3V */
1057 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
1058 /* EOImode */
1059 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
1060 /* CBPR */
1061 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
1062
1063 vcpu_set_reg(vcpu, rt, val);
1064 }
1065
__vgic_v3_write_ctlr(struct kvm_vcpu * vcpu,u32 vmcr,int rt)1066 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1067 {
1068 u32 val = vcpu_get_reg(vcpu, rt);
1069
1070 if (val & ICC_CTLR_EL1_CBPR_MASK)
1071 vmcr |= ICH_VMCR_CBPR_MASK;
1072 else
1073 vmcr &= ~ICH_VMCR_CBPR_MASK;
1074
1075 if (val & ICC_CTLR_EL1_EOImode_MASK)
1076 vmcr |= ICH_VMCR_EOIM_MASK;
1077 else
1078 vmcr &= ~ICH_VMCR_EOIM_MASK;
1079
1080 write_gicreg(vmcr, ICH_VMCR_EL2);
1081 }
1082
__vgic_v3_check_trap_forwarding(struct kvm_vcpu * vcpu,u32 sysreg,bool is_read)1083 static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu,
1084 u32 sysreg, bool is_read)
1085 {
1086 u64 ich_hcr;
1087
1088 if (!is_nested_ctxt(vcpu))
1089 return false;
1090
1091 ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);
1092
1093 switch (sysreg) {
1094 case SYS_ICC_IGRPEN0_EL1:
1095 if (is_read &&
1096 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
1097 return true;
1098
1099 if (!is_read &&
1100 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
1101 return true;
1102
1103 fallthrough;
1104
1105 case SYS_ICC_AP0Rn_EL1(0):
1106 case SYS_ICC_AP0Rn_EL1(1):
1107 case SYS_ICC_AP0Rn_EL1(2):
1108 case SYS_ICC_AP0Rn_EL1(3):
1109 case SYS_ICC_BPR0_EL1:
1110 case SYS_ICC_EOIR0_EL1:
1111 case SYS_ICC_HPPIR0_EL1:
1112 case SYS_ICC_IAR0_EL1:
1113 return ich_hcr & ICH_HCR_EL2_TALL0;
1114
1115 case SYS_ICC_IGRPEN1_EL1:
1116 if (is_read &&
1117 (__vcpu_sys_reg(vcpu, HFGRTR_EL2) & HFGRTR_EL2_ICC_IGRPENn_EL1))
1118 return true;
1119
1120 if (!is_read &&
1121 (__vcpu_sys_reg(vcpu, HFGWTR_EL2) & HFGWTR_EL2_ICC_IGRPENn_EL1))
1122 return true;
1123
1124 fallthrough;
1125
1126 case SYS_ICC_AP1Rn_EL1(0):
1127 case SYS_ICC_AP1Rn_EL1(1):
1128 case SYS_ICC_AP1Rn_EL1(2):
1129 case SYS_ICC_AP1Rn_EL1(3):
1130 case SYS_ICC_BPR1_EL1:
1131 case SYS_ICC_EOIR1_EL1:
1132 case SYS_ICC_HPPIR1_EL1:
1133 case SYS_ICC_IAR1_EL1:
1134 return ich_hcr & ICH_HCR_EL2_TALL1;
1135
1136 case SYS_ICC_DIR_EL1:
1137 if (ich_hcr & ICH_HCR_EL2_TDIR)
1138 return true;
1139
1140 fallthrough;
1141
1142 case SYS_ICC_RPR_EL1:
1143 case SYS_ICC_CTLR_EL1:
1144 case SYS_ICC_PMR_EL1:
1145 return ich_hcr & ICH_HCR_EL2_TC;
1146
1147 default:
1148 return false;
1149 }
1150 }
1151
__vgic_v3_perform_cpuif_access(struct kvm_vcpu * vcpu)1152 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1153 {
1154 int rt;
1155 u64 esr;
1156 u32 vmcr;
1157 void (*fn)(struct kvm_vcpu *, u32, int);
1158 bool is_read;
1159 u32 sysreg;
1160
1161 if (kern_hyp_va(vcpu->kvm)->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V3)
1162 return 0;
1163
1164 esr = kvm_vcpu_get_esr(vcpu);
1165 if (vcpu_mode_is_32bit(vcpu)) {
1166 if (!kvm_condition_valid(vcpu)) {
1167 __kvm_skip_instr(vcpu);
1168 return 1;
1169 }
1170
1171 sysreg = esr_cp15_to_sysreg(esr);
1172 } else {
1173 sysreg = esr_sys64_to_sysreg(esr);
1174 }
1175
1176 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1177
1178 if (__vgic_v3_check_trap_forwarding(vcpu, sysreg, is_read))
1179 return 0;
1180
1181 switch (sysreg) {
1182 case SYS_ICC_IAR0_EL1:
1183 case SYS_ICC_IAR1_EL1:
1184 if (unlikely(!is_read))
1185 return 0;
1186 fn = __vgic_v3_read_iar;
1187 break;
1188 case SYS_ICC_EOIR0_EL1:
1189 case SYS_ICC_EOIR1_EL1:
1190 if (unlikely(is_read))
1191 return 0;
1192 fn = __vgic_v3_write_eoir;
1193 break;
1194 case SYS_ICC_IGRPEN1_EL1:
1195 if (is_read)
1196 fn = __vgic_v3_read_igrpen1;
1197 else
1198 fn = __vgic_v3_write_igrpen1;
1199 break;
1200 case SYS_ICC_BPR1_EL1:
1201 if (is_read)
1202 fn = __vgic_v3_read_bpr1;
1203 else
1204 fn = __vgic_v3_write_bpr1;
1205 break;
1206 case SYS_ICC_AP0Rn_EL1(0):
1207 case SYS_ICC_AP1Rn_EL1(0):
1208 if (is_read)
1209 fn = __vgic_v3_read_apxr0;
1210 else
1211 fn = __vgic_v3_write_apxr0;
1212 break;
1213 case SYS_ICC_AP0Rn_EL1(1):
1214 case SYS_ICC_AP1Rn_EL1(1):
1215 if (is_read)
1216 fn = __vgic_v3_read_apxr1;
1217 else
1218 fn = __vgic_v3_write_apxr1;
1219 break;
1220 case SYS_ICC_AP0Rn_EL1(2):
1221 case SYS_ICC_AP1Rn_EL1(2):
1222 if (is_read)
1223 fn = __vgic_v3_read_apxr2;
1224 else
1225 fn = __vgic_v3_write_apxr2;
1226 break;
1227 case SYS_ICC_AP0Rn_EL1(3):
1228 case SYS_ICC_AP1Rn_EL1(3):
1229 if (is_read)
1230 fn = __vgic_v3_read_apxr3;
1231 else
1232 fn = __vgic_v3_write_apxr3;
1233 break;
1234 case SYS_ICC_HPPIR0_EL1:
1235 case SYS_ICC_HPPIR1_EL1:
1236 if (unlikely(!is_read))
1237 return 0;
1238 fn = __vgic_v3_read_hppir;
1239 break;
1240 case SYS_ICC_IGRPEN0_EL1:
1241 if (is_read)
1242 fn = __vgic_v3_read_igrpen0;
1243 else
1244 fn = __vgic_v3_write_igrpen0;
1245 break;
1246 case SYS_ICC_BPR0_EL1:
1247 if (is_read)
1248 fn = __vgic_v3_read_bpr0;
1249 else
1250 fn = __vgic_v3_write_bpr0;
1251 break;
1252 case SYS_ICC_DIR_EL1:
1253 if (unlikely(is_read))
1254 return 0;
1255 fn = __vgic_v3_write_dir;
1256 break;
1257 case SYS_ICC_RPR_EL1:
1258 if (unlikely(!is_read))
1259 return 0;
1260 fn = __vgic_v3_read_rpr;
1261 break;
1262 case SYS_ICC_CTLR_EL1:
1263 if (is_read)
1264 fn = __vgic_v3_read_ctlr;
1265 else
1266 fn = __vgic_v3_write_ctlr;
1267 break;
1268 case SYS_ICC_PMR_EL1:
1269 if (is_read)
1270 fn = __vgic_v3_read_pmr;
1271 else
1272 fn = __vgic_v3_write_pmr;
1273 break;
1274 default:
1275 return 0;
1276 }
1277
1278 vmcr = __vgic_v3_read_vmcr();
1279 rt = kvm_vcpu_sys_get_rt(vcpu);
1280 fn(vcpu, vmcr, rt);
1281
1282 __kvm_skip_instr(vcpu);
1283
1284 return 1;
1285 }
1286