1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2015, 2016 ARM Ltd.
4 */
5
6 #include <linux/uaccess.h>
7 #include <linux/interrupt.h>
8 #include <linux/cpu.h>
9 #include <linux/kvm_host.h>
10 #include <kvm/arm_vgic.h>
11 #include <asm/kvm_emulate.h>
12 #include <asm/kvm_mmu.h>
13 #include "vgic.h"
14
15 /*
16 * Initialization rules: there are multiple stages to the vgic
17 * initialization, both for the distributor and the CPU interfaces. The basic
18 * idea is that even though the VGIC is not functional or not requested from
19 * user space, the critical path of the run loop can still call VGIC functions
20 * that just won't do anything, without them having to check additional
21 * initialization flags to ensure they don't look at uninitialized data
22 * structures.
23 *
24 * Distributor:
25 *
26 * - kvm_vgic_early_init(): initialization of static data that doesn't
27 * depend on any sizing information or emulation type. No allocation
28 * is allowed there.
29 *
30 * - vgic_init(): allocation and initialization of the generic data
31 * structures that depend on sizing information (number of CPUs,
32 * number of interrupts). Also initializes the vcpu specific data
33 * structures. Can be executed lazily for GICv2.
34 *
35 * CPU Interface:
36 *
37 * - kvm_vgic_vcpu_init(): initialization of static data that doesn't depend
38 * on any sizing information. Private interrupts are allocated if not
39 * already allocated at vgic-creation time.
40 */
41
42 /* EARLY INIT */
43
44 /**
45 * kvm_vgic_early_init() - Initialize static VGIC VCPU data structures
46 * @kvm: The VM whose VGIC districutor should be initialized
47 *
48 * Only do initialization of static structures that don't require any
49 * allocation or sizing information from userspace. vgic_init() called
50 * kvm_vgic_dist_init() which takes care of the rest.
51 */
kvm_vgic_early_init(struct kvm * kvm)52 void kvm_vgic_early_init(struct kvm *kvm)
53 {
54 struct vgic_dist *dist = &kvm->arch.vgic;
55
56 xa_init_flags(&dist->lpi_xa, XA_FLAGS_LOCK_IRQ);
57 }
58
59 /* CREATION */
60
61 static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu, u32 type);
62
63 /**
64 * kvm_vgic_create: triggered by the instantiation of the VGIC device by
65 * user space, either through the legacy KVM_CREATE_IRQCHIP ioctl (v2 only)
66 * or through the generic KVM_CREATE_DEVICE API ioctl.
67 * irqchip_in_kernel() tells you if this function succeeded or not.
68 * @kvm: kvm struct pointer
69 * @type: KVM_DEV_TYPE_ARM_VGIC_V[235]
70 */
kvm_vgic_create(struct kvm * kvm,u32 type)71 int kvm_vgic_create(struct kvm *kvm, u32 type)
72 {
73 struct kvm_vcpu *vcpu;
74 unsigned long i;
75 int ret;
76
77 /*
78 * This function is also called by the KVM_CREATE_IRQCHIP handler,
79 * which had no chance yet to check the availability of the GICv2
80 * emulation. So check this here again. KVM_CREATE_DEVICE does
81 * the proper checks already.
82 */
83 if (type == KVM_DEV_TYPE_ARM_VGIC_V2 &&
84 !kvm_vgic_global_state.can_emulate_gicv2)
85 return -ENODEV;
86
87 /*
88 * Ensure mutual exclusion with vCPU creation and any vCPU ioctls by:
89 *
90 * - Holding kvm->lock to prevent KVM_CREATE_VCPU from reaching
91 * kvm_arch_vcpu_precreate() and ensuring created_vcpus is stable.
92 * This alone is insufficient, as kvm_vm_ioctl_create_vcpu() drops
93 * the kvm->lock before completing the vCPU creation.
94 */
95 lockdep_assert_held(&kvm->lock);
96
97 /*
98 * - Acquiring the vCPU mutex for every *online* vCPU to prevent
99 * concurrent vCPU ioctls for vCPUs already visible to userspace.
100 */
101 ret = -EBUSY;
102 if (kvm_trylock_all_vcpus(kvm))
103 return ret;
104
105 /*
106 * - Taking the config_lock which protects VGIC data structures such
107 * as the per-vCPU arrays of private IRQs (SGIs, PPIs).
108 */
109 mutex_lock(&kvm->arch.config_lock);
110
111 /*
112 * - Bailing on the entire thing if a vCPU is in the middle of creation,
113 * dropped the kvm->lock, but hasn't reached kvm_arch_vcpu_create().
114 *
115 * The whole combination of this guarantees that no vCPU can get into
116 * KVM with a VGIC configuration inconsistent with the VM's VGIC.
117 */
118 if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
119 goto out_unlock;
120
121 if (irqchip_in_kernel(kvm)) {
122 ret = -EEXIST;
123 goto out_unlock;
124 }
125
126 kvm_for_each_vcpu(i, vcpu, kvm) {
127 if (vcpu_has_run_once(vcpu))
128 goto out_unlock;
129 }
130 ret = 0;
131
132 if (type == KVM_DEV_TYPE_ARM_VGIC_V2)
133 kvm->max_vcpus = VGIC_V2_MAX_CPUS;
134 else if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
135 kvm->max_vcpus = VGIC_V3_MAX_CPUS;
136 else if (type == KVM_DEV_TYPE_ARM_VGIC_V5)
137 kvm->max_vcpus = min(VGIC_V5_MAX_CPUS,
138 kvm_vgic_global_state.max_gic_vcpus);
139
140 if (atomic_read(&kvm->online_vcpus) > kvm->max_vcpus) {
141 ret = -E2BIG;
142 goto out_unlock;
143 }
144
145 kvm->arch.vgic.in_kernel = true;
146 kvm->arch.vgic.vgic_model = type;
147 kvm->arch.vgic.implementation_rev = KVM_VGIC_IMP_REV_LATEST;
148 kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
149
150 switch (type) {
151 case KVM_DEV_TYPE_ARM_VGIC_V2:
152 kvm->arch.vgic.vgic_cpu_base = VGIC_ADDR_UNDEF;
153 break;
154 case KVM_DEV_TYPE_ARM_VGIC_V3:
155 INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
156 break;
157 }
158
159 /*
160 * We've now created the GIC. Update the system register state
161 * to accurately reflect what we've created.
162 */
163 kvm_vgic_finalize_idregs(kvm);
164
165 kvm_for_each_vcpu(i, vcpu, kvm) {
166 ret = vgic_allocate_private_irqs_locked(vcpu, type);
167 if (ret)
168 break;
169 }
170
171 if (ret) {
172 kvm_for_each_vcpu(i, vcpu, kvm) {
173 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
174 kfree(vgic_cpu->private_irqs);
175 vgic_cpu->private_irqs = NULL;
176 }
177
178 kvm->arch.vgic.vgic_model = 0;
179 goto out_unlock;
180 }
181
182 if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
183 kvm->arch.vgic.nassgicap = system_supports_direct_sgis();
184
185 /*
186 * We now know that we have a GICv5. The Arch Timer PPI interrupts may
187 * have been initialised at this stage, but will have done so assuming
188 * that we have an older GIC, meaning that the IntIDs won't be
189 * correct. We init them again, and this time they will be correct.
190 */
191 if (type == KVM_DEV_TYPE_ARM_VGIC_V5)
192 kvm_timer_init_vm(kvm);
193
194 out_unlock:
195 mutex_unlock(&kvm->arch.config_lock);
196 kvm_unlock_all_vcpus(kvm);
197 return ret;
198 }
199
200 /* INIT/DESTROY */
201
202 /**
203 * kvm_vgic_dist_init: initialize the dist data structures
204 * @kvm: kvm struct pointer
205 * @nr_spis: number of spis, frozen by caller
206 */
kvm_vgic_dist_init(struct kvm * kvm,unsigned int nr_spis)207 static int kvm_vgic_dist_init(struct kvm *kvm, unsigned int nr_spis)
208 {
209 struct vgic_dist *dist = &kvm->arch.vgic;
210 struct kvm_vcpu *vcpu0 = kvm_get_vcpu(kvm, 0);
211 int i;
212
213 dist->active_spis = (atomic_t)ATOMIC_INIT(0);
214 dist->spis = kzalloc_objs(struct vgic_irq, nr_spis, GFP_KERNEL_ACCOUNT);
215 if (!dist->spis)
216 return -ENOMEM;
217
218 /*
219 * In the following code we do not take the irq struct lock since
220 * no other action on irq structs can happen while the VGIC is
221 * not initialized yet:
222 * If someone wants to inject an interrupt or does a MMIO access, we
223 * require prior initialization in case of a virtual GICv3 or trigger
224 * initialization when using a virtual GICv2.
225 */
226 for (i = 0; i < nr_spis; i++) {
227 struct vgic_irq *irq = &dist->spis[i];
228
229 irq->intid = i + VGIC_NR_PRIVATE_IRQS;
230 INIT_LIST_HEAD(&irq->ap_list);
231 raw_spin_lock_init(&irq->irq_lock);
232 irq->vcpu = NULL;
233 irq->target_vcpu = vcpu0;
234 refcount_set(&irq->refcount, 0);
235 switch (dist->vgic_model) {
236 case KVM_DEV_TYPE_ARM_VGIC_V2:
237 irq->targets = 0;
238 irq->group = 0;
239 break;
240 case KVM_DEV_TYPE_ARM_VGIC_V3:
241 irq->mpidr = 0;
242 irq->group = 1;
243 break;
244 default:
245 kfree(dist->spis);
246 dist->spis = NULL;
247 return -EINVAL;
248 }
249 }
250 return 0;
251 }
252
253 /* Default GICv3 Maintenance Interrupt INTID, as per SBSA */
254 #define DEFAULT_MI_INTID 25
255
kvm_vgic_vcpu_nv_init(struct kvm_vcpu * vcpu)256 int kvm_vgic_vcpu_nv_init(struct kvm_vcpu *vcpu)
257 {
258 int ret;
259
260 guard(mutex)(&vcpu->kvm->arch.config_lock);
261
262 /*
263 * Matching the tradition established with the timers, provide
264 * a default PPI for the maintenance interrupt. It makes
265 * things easier to reason about.
266 */
267 if (vcpu->kvm->arch.vgic.mi_intid == 0)
268 vcpu->kvm->arch.vgic.mi_intid = DEFAULT_MI_INTID;
269 ret = kvm_vgic_set_owner(vcpu, vcpu->kvm->arch.vgic.mi_intid, vcpu);
270
271 return ret;
272 }
273
vgic_allocate_private_irq(struct kvm_vcpu * vcpu,int i,u32 type)274 static void vgic_allocate_private_irq(struct kvm_vcpu *vcpu, int i, u32 type)
275 {
276 struct vgic_irq *irq = &vcpu->arch.vgic_cpu.private_irqs[i];
277
278 INIT_LIST_HEAD(&irq->ap_list);
279 raw_spin_lock_init(&irq->irq_lock);
280 irq->vcpu = NULL;
281 irq->target_vcpu = vcpu;
282 refcount_set(&irq->refcount, 0);
283
284 irq->intid = i;
285 if (vgic_irq_is_sgi(i)) {
286 /* SGIs */
287 irq->enabled = 1;
288 irq->config = VGIC_CONFIG_EDGE;
289 } else {
290 /* PPIs */
291 irq->config = VGIC_CONFIG_LEVEL;
292 }
293
294 switch (type) {
295 case KVM_DEV_TYPE_ARM_VGIC_V3:
296 irq->group = 1;
297 irq->mpidr = kvm_vcpu_get_mpidr_aff(vcpu);
298 break;
299 case KVM_DEV_TYPE_ARM_VGIC_V2:
300 irq->group = 0;
301 irq->targets = BIT(vcpu->vcpu_id);
302 break;
303 }
304 }
305
vgic_v5_allocate_private_irq(struct kvm_vcpu * vcpu,int i,u32 type)306 static void vgic_v5_allocate_private_irq(struct kvm_vcpu *vcpu, int i, u32 type)
307 {
308 struct vgic_irq *irq = &vcpu->arch.vgic_cpu.private_irqs[i];
309 u32 intid = vgic_v5_make_ppi(i);
310
311 INIT_LIST_HEAD(&irq->ap_list);
312 raw_spin_lock_init(&irq->irq_lock);
313 irq->vcpu = NULL;
314 irq->target_vcpu = vcpu;
315 refcount_set(&irq->refcount, 0);
316
317 irq->intid = intid;
318
319 /* The only Edge architected PPI is the SW_PPI */
320 if (i == GICV5_ARCH_PPI_SW_PPI)
321 irq->config = VGIC_CONFIG_EDGE;
322 else
323 irq->config = VGIC_CONFIG_LEVEL;
324
325 /* Register the GICv5-specific PPI ops */
326 vgic_v5_set_ppi_ops(vcpu, intid);
327 }
328
vgic_allocate_private_irqs_locked(struct kvm_vcpu * vcpu,u32 type)329 static int vgic_allocate_private_irqs_locked(struct kvm_vcpu *vcpu, u32 type)
330 {
331 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
332 u32 num_private_irqs;
333 int i;
334
335 lockdep_assert_held(&vcpu->kvm->arch.config_lock);
336
337 if (vgic_cpu->private_irqs)
338 return 0;
339
340 if (vgic_is_v5(vcpu->kvm))
341 num_private_irqs = VGIC_V5_NR_PRIVATE_IRQS;
342 else
343 num_private_irqs = VGIC_NR_PRIVATE_IRQS;
344
345 vgic_cpu->private_irqs = kzalloc_objs(struct vgic_irq,
346 num_private_irqs,
347 GFP_KERNEL_ACCOUNT);
348
349 if (!vgic_cpu->private_irqs)
350 return -ENOMEM;
351
352 /*
353 * Enable and configure all SGIs to be edge-triggered and
354 * configure all PPIs as level-triggered.
355 */
356 for (i = 0; i < num_private_irqs; i++) {
357 if (vgic_is_v5(vcpu->kvm))
358 vgic_v5_allocate_private_irq(vcpu, i, type);
359 else
360 vgic_allocate_private_irq(vcpu, i, type);
361 }
362
363 return 0;
364 }
365
vgic_allocate_private_irqs(struct kvm_vcpu * vcpu,u32 type)366 static int vgic_allocate_private_irqs(struct kvm_vcpu *vcpu, u32 type)
367 {
368 int ret;
369
370 mutex_lock(&vcpu->kvm->arch.config_lock);
371 ret = vgic_allocate_private_irqs_locked(vcpu, type);
372 mutex_unlock(&vcpu->kvm->arch.config_lock);
373
374 return ret;
375 }
376
377 /**
378 * kvm_vgic_vcpu_init() - Initialize static VGIC VCPU data
379 * structures and register VCPU-specific KVM iodevs
380 *
381 * @vcpu: pointer to the VCPU being created and initialized
382 *
383 * Only do initialization, but do not actually enable the
384 * VGIC CPU interface
385 */
kvm_vgic_vcpu_init(struct kvm_vcpu * vcpu)386 int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)
387 {
388 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
389 struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
390 int ret = 0;
391
392 vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
393
394 INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
395 raw_spin_lock_init(&vgic_cpu->ap_list_lock);
396 atomic_set(&vgic_cpu->vgic_v3.its_vpe.vlpi_count, 0);
397
398 if (!irqchip_in_kernel(vcpu->kvm))
399 return 0;
400
401 ret = vgic_allocate_private_irqs(vcpu, dist->vgic_model);
402 if (ret)
403 return ret;
404
405 /*
406 * If we are creating a VCPU with a GICv3 we must also register the
407 * KVM io device for the redistributor that belongs to this VCPU.
408 */
409 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
410 mutex_lock(&vcpu->kvm->slots_lock);
411 ret = vgic_register_redist_iodev(vcpu);
412 mutex_unlock(&vcpu->kvm->slots_lock);
413 }
414 return ret;
415 }
416
kvm_vgic_vcpu_reset(struct kvm_vcpu * vcpu)417 static void kvm_vgic_vcpu_reset(struct kvm_vcpu *vcpu)
418 {
419 const struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
420
421 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V5)
422 vgic_v5_reset(vcpu);
423 else if (kvm_vgic_global_state.type == VGIC_V2)
424 vgic_v2_reset(vcpu);
425 else
426 vgic_v3_reset(vcpu);
427 }
428
429 /*
430 * vgic_init: allocates and initializes dist and vcpu data structures
431 * depending on two dimensioning parameters:
432 * - the number of spis
433 * - the number of vcpus
434 * The function is generally called when nr_spis has been explicitly set
435 * by the guest through the KVM DEVICE API. If not nr_spis is set to 256.
436 * vgic_initialized() returns true when this function has succeeded.
437 */
vgic_init(struct kvm * kvm)438 int vgic_init(struct kvm *kvm)
439 {
440 struct vgic_dist *dist = &kvm->arch.vgic;
441 struct kvm_vcpu *vcpu;
442 int ret = 0;
443 unsigned long idx;
444
445 lockdep_assert_held(&kvm->arch.config_lock);
446
447 if (vgic_initialized(kvm))
448 return 0;
449
450 /* Are we also in the middle of creating a VCPU? */
451 if (kvm->created_vcpus != atomic_read(&kvm->online_vcpus))
452 return -EBUSY;
453
454 if (!vgic_is_v5(kvm)) {
455 /* freeze the number of spis */
456 if (!dist->nr_spis)
457 dist->nr_spis = VGIC_NR_IRQS_LEGACY - VGIC_NR_PRIVATE_IRQS;
458
459 ret = kvm_vgic_dist_init(kvm, dist->nr_spis);
460 if (ret)
461 return ret;
462
463 /*
464 * Ensure vPEs are allocated if direct IRQ injection (e.g. vSGIs,
465 * vLPIs) is supported.
466 */
467 if (vgic_supports_direct_irqs(kvm)) {
468 ret = vgic_v4_init(kvm);
469 if (ret)
470 return ret;
471 }
472 } else {
473 ret = vgic_v5_init(kvm);
474 if (ret)
475 return ret;
476 }
477
478 kvm_for_each_vcpu(idx, vcpu, kvm)
479 kvm_vgic_vcpu_reset(vcpu);
480
481 ret = kvm_vgic_setup_default_irq_routing(kvm);
482 if (ret)
483 return ret;
484
485 vgic_debug_init(kvm);
486 dist->initialized = true;
487
488 return 0;
489 }
490
kvm_vgic_dist_destroy(struct kvm * kvm)491 static void kvm_vgic_dist_destroy(struct kvm *kvm)
492 {
493 struct vgic_dist *dist = &kvm->arch.vgic;
494 struct vgic_redist_region *rdreg, *next;
495
496 dist->ready = false;
497 dist->initialized = false;
498
499 kfree(dist->spis);
500 dist->spis = NULL;
501 dist->nr_spis = 0;
502 dist->vgic_dist_base = VGIC_ADDR_UNDEF;
503
504 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
505 list_for_each_entry_safe(rdreg, next, &dist->rd_regions, list)
506 vgic_v3_free_redist_region(kvm, rdreg);
507 INIT_LIST_HEAD(&dist->rd_regions);
508 } else {
509 dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
510 }
511
512 if (vgic_supports_direct_irqs(kvm))
513 vgic_v4_teardown(kvm);
514
515 xa_destroy(&dist->lpi_xa);
516 }
517
__kvm_vgic_vcpu_destroy(struct kvm_vcpu * vcpu)518 static void __kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
519 {
520 struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
521
522 /*
523 * Retire all pending LPIs on this vcpu anyway as we're
524 * going to destroy it.
525 */
526 vgic_flush_pending_lpis(vcpu);
527
528 INIT_LIST_HEAD(&vgic_cpu->ap_list_head);
529 kfree(vgic_cpu->private_irqs);
530 vgic_cpu->private_irqs = NULL;
531
532 if (vcpu->kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
533 /*
534 * If this vCPU is being destroyed because of a failed creation
535 * then unregister the redistributor to avoid leaving behind a
536 * dangling pointer to the vCPU struct.
537 *
538 * vCPUs that have been successfully created (i.e. added to
539 * kvm->vcpu_array) get unregistered in kvm_vgic_destroy(), as
540 * this function gets called while holding kvm->arch.config_lock
541 * in the VM teardown path and would otherwise introduce a lock
542 * inversion w.r.t. kvm->srcu.
543 *
544 * vCPUs that failed creation are torn down outside of the
545 * kvm->arch.config_lock and do not get unregistered in
546 * kvm_vgic_destroy(), meaning it is both safe and necessary to
547 * do so here.
548 */
549 if (kvm_get_vcpu_by_id(vcpu->kvm, vcpu->vcpu_id) != vcpu)
550 vgic_unregister_redist_iodev(vcpu);
551
552 vgic_cpu->rd_iodev.base_addr = VGIC_ADDR_UNDEF;
553 }
554 }
555
kvm_vgic_vcpu_destroy(struct kvm_vcpu * vcpu)556 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu)
557 {
558 struct kvm *kvm = vcpu->kvm;
559
560 mutex_lock(&kvm->slots_lock);
561 __kvm_vgic_vcpu_destroy(vcpu);
562 mutex_unlock(&kvm->slots_lock);
563 }
564
kvm_vgic_destroy(struct kvm * kvm)565 void kvm_vgic_destroy(struct kvm *kvm)
566 {
567 struct kvm_vcpu *vcpu;
568 unsigned long i;
569
570 mutex_lock(&kvm->slots_lock);
571 mutex_lock(&kvm->arch.config_lock);
572
573 vgic_debug_destroy(kvm);
574
575 kvm_for_each_vcpu(i, vcpu, kvm)
576 __kvm_vgic_vcpu_destroy(vcpu);
577
578 kvm_vgic_dist_destroy(kvm);
579
580 mutex_unlock(&kvm->arch.config_lock);
581
582 if (kvm->arch.vgic.vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3)
583 kvm_for_each_vcpu(i, vcpu, kvm)
584 vgic_unregister_redist_iodev(vcpu);
585
586 mutex_unlock(&kvm->slots_lock);
587 }
588
589 /**
590 * vgic_lazy_init: Lazy init is only allowed if the GIC exposed to the guest
591 * is a GICv2. A GICv3 must be explicitly initialized by userspace using the
592 * KVM_DEV_ARM_VGIC_GRP_CTRL KVM_DEVICE group.
593 * @kvm: kvm struct pointer
594 */
vgic_lazy_init(struct kvm * kvm)595 int vgic_lazy_init(struct kvm *kvm)
596 {
597 int ret = 0;
598
599 if (unlikely(!vgic_initialized(kvm))) {
600 /*
601 * We only provide the automatic initialization of the VGIC
602 * for the legacy case of a GICv2. Any other type must
603 * be explicitly initialized once setup with the respective
604 * KVM device call.
605 */
606 if (kvm->arch.vgic.vgic_model != KVM_DEV_TYPE_ARM_VGIC_V2)
607 return -EBUSY;
608
609 mutex_lock(&kvm->arch.config_lock);
610 ret = vgic_init(kvm);
611 mutex_unlock(&kvm->arch.config_lock);
612 }
613
614 return ret;
615 }
616
617 /* RESOURCE MAPPING */
618
619 /**
620 * kvm_vgic_map_resources - map the MMIO regions
621 * @kvm: kvm struct pointer
622 *
623 * Map the MMIO regions depending on the VGIC model exposed to the guest
624 * called on the first VCPU run.
625 * Also map the virtual CPU interface into the VM.
626 * v2 calls vgic_init() if not already done.
627 * v3 and derivatives return an error if the VGIC is not initialized.
628 */
kvm_vgic_map_resources(struct kvm * kvm)629 int kvm_vgic_map_resources(struct kvm *kvm)
630 {
631 struct vgic_dist *dist = &kvm->arch.vgic;
632 bool needs_dist = true;
633 enum vgic_type type;
634 gpa_t dist_base;
635 int ret = 0;
636
637 if (likely(smp_load_acquire(&dist->ready)))
638 return 0;
639
640 mutex_lock(&kvm->slots_lock);
641 mutex_lock(&kvm->arch.config_lock);
642 if (dist->ready)
643 goto out;
644
645 if (!irqchip_in_kernel(kvm))
646 goto out;
647
648 if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V2) {
649 ret = vgic_v2_map_resources(kvm);
650 type = VGIC_V2;
651 } else if (dist->vgic_model == KVM_DEV_TYPE_ARM_VGIC_V3) {
652 ret = vgic_v3_map_resources(kvm);
653 type = VGIC_V3;
654 } else {
655 ret = vgic_v5_map_resources(kvm);
656 type = VGIC_V5;
657 needs_dist = false;
658 }
659
660 if (ret)
661 goto out;
662
663 if (needs_dist) {
664 dist_base = dist->vgic_dist_base;
665 mutex_unlock(&kvm->arch.config_lock);
666
667 ret = vgic_register_dist_iodev(kvm, dist_base, type);
668 if (ret) {
669 kvm_err("Unable to register VGIC dist MMIO regions\n");
670 goto out_slots;
671 }
672 } else {
673 mutex_unlock(&kvm->arch.config_lock);
674 }
675
676 smp_store_release(&dist->ready, true);
677 goto out_slots;
678 out:
679 mutex_unlock(&kvm->arch.config_lock);
680 out_slots:
681 if (ret)
682 kvm_vm_dead(kvm);
683
684 mutex_unlock(&kvm->slots_lock);
685
686 return ret;
687 }
688
kvm_vgic_finalize_idregs(struct kvm * kvm)689 void kvm_vgic_finalize_idregs(struct kvm *kvm)
690 {
691 u32 type = kvm->arch.vgic.vgic_model;
692 u64 aa64pfr0, aa64pfr2, pfr1;
693
694 aa64pfr0 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1) & ~ID_AA64PFR0_EL1_GIC;
695 aa64pfr2 = kvm_read_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1) & ~ID_AA64PFR2_EL1_GCIE;
696 pfr1 = kvm_read_vm_id_reg(kvm, SYS_ID_PFR1_EL1) & ~ID_PFR1_EL1_GIC;
697
698 switch (type) {
699 case KVM_DEV_TYPE_ARM_VGIC_V2:
700 break;
701 case KVM_DEV_TYPE_ARM_VGIC_V3:
702 aa64pfr0 |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
703 if (kvm_supports_32bit_el0())
704 pfr1 |= SYS_FIELD_PREP_ENUM(ID_PFR1_EL1, GIC, GICv3);
705 break;
706 case KVM_DEV_TYPE_ARM_VGIC_V5:
707 aa64pfr2 |= SYS_FIELD_PREP_ENUM(ID_AA64PFR2_EL1, GCIE, IMP);
708 break;
709 default:
710 WARN_ONCE(1, "Unknown VGIC type!!!\n");
711 }
712
713 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR0_EL1, aa64pfr0);
714 kvm_set_vm_id_reg(kvm, SYS_ID_AA64PFR2_EL1, aa64pfr2);
715 kvm_set_vm_id_reg(kvm, SYS_ID_PFR1_EL1, pfr1);
716 }
717
718 /* GENERIC PROBE */
719
kvm_vgic_cpu_up(void)720 void kvm_vgic_cpu_up(void)
721 {
722 enable_percpu_irq(kvm_vgic_global_state.maint_irq, 0);
723 }
724
725
kvm_vgic_cpu_down(void)726 void kvm_vgic_cpu_down(void)
727 {
728 disable_percpu_irq(kvm_vgic_global_state.maint_irq);
729 }
730
vgic_maintenance_handler(int irq,void * data)731 static irqreturn_t vgic_maintenance_handler(int irq, void *data)
732 {
733 struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)data;
734
735 /*
736 * We cannot rely on the vgic maintenance interrupt to be
737 * delivered synchronously. This means we can only use it to
738 * exit the VM, and we perform the handling of EOIed
739 * interrupts on the exit path (see vgic_fold_lr_state).
740 *
741 * Of course, NV throws a wrench in this plan, and needs
742 * something special.
743 */
744 if (vcpu && vgic_state_is_nested(vcpu))
745 vgic_v3_handle_nested_maint_irq(vcpu);
746
747 return IRQ_HANDLED;
748 }
749
750 static struct gic_kvm_info *gic_kvm_info;
751
vgic_set_kvm_info(const struct gic_kvm_info * info)752 void __init vgic_set_kvm_info(const struct gic_kvm_info *info)
753 {
754 BUG_ON(gic_kvm_info != NULL);
755 gic_kvm_info = kmalloc_obj(*gic_kvm_info);
756 if (gic_kvm_info)
757 *gic_kvm_info = *info;
758 }
759
760 /**
761 * kvm_vgic_init_cpu_hardware - initialize the GIC VE hardware
762 *
763 * For a specific CPU, initialize the GIC VE hardware.
764 */
kvm_vgic_init_cpu_hardware(void)765 void kvm_vgic_init_cpu_hardware(void)
766 {
767 BUG_ON(preemptible());
768
769 /*
770 * We want to make sure the list registers start out clear so that we
771 * only have the program the used registers.
772 */
773 if (kvm_vgic_global_state.type == VGIC_V2) {
774 vgic_v2_init_lrs();
775 } else if (kvm_vgic_global_state.type == VGIC_V3 ||
776 kvm_vgic_global_state.has_gcie_v3_compat) {
777 kvm_call_hyp(__vgic_v3_init_lrs);
778 }
779 }
780
781 /**
782 * kvm_vgic_hyp_init: populates the kvm_vgic_global_state variable
783 * according to the host GIC model. Accordingly calls either
784 * vgic_v2/v3_probe which registers the KVM_DEVICE that can be
785 * instantiated by a guest later on .
786 */
kvm_vgic_hyp_init(void)787 int kvm_vgic_hyp_init(void)
788 {
789 bool has_mask;
790 int ret;
791
792 if (!gic_kvm_info)
793 return -ENODEV;
794
795 has_mask = !gic_kvm_info->no_maint_irq_mask;
796
797 if (has_mask && !gic_kvm_info->maint_irq) {
798 kvm_err("No vgic maintenance irq\n");
799 return -ENXIO;
800 }
801
802 /*
803 * If we get one of these oddball non-GICs, taint the kernel,
804 * as we have no idea of how they *really* behave.
805 */
806 if (gic_kvm_info->no_hw_deactivation) {
807 kvm_info("Non-architectural vgic, tainting kernel\n");
808 add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
809 kvm_vgic_global_state.no_hw_deactivation = true;
810 }
811
812 switch (gic_kvm_info->type) {
813 case GIC_V2:
814 ret = vgic_v2_probe(gic_kvm_info);
815 break;
816 case GIC_V3:
817 ret = vgic_v3_probe(gic_kvm_info);
818 if (!ret) {
819 static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
820 kvm_info("GIC system register CPU interface enabled\n");
821 }
822 break;
823 case GIC_V5:
824 ret = vgic_v5_probe(gic_kvm_info);
825 break;
826 default:
827 ret = -ENODEV;
828 }
829
830 kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq;
831
832 kfree(gic_kvm_info);
833 gic_kvm_info = NULL;
834
835 if (ret)
836 return ret;
837
838 if (!has_mask && !kvm_vgic_global_state.maint_irq)
839 return 0;
840
841 ret = request_percpu_irq(kvm_vgic_global_state.maint_irq,
842 vgic_maintenance_handler,
843 "vgic", kvm_get_running_vcpus());
844 if (ret) {
845 kvm_err("Cannot register interrupt %d\n",
846 kvm_vgic_global_state.maint_irq);
847 return ret;
848 }
849
850 kvm_info("vgic interrupt IRQ%d\n", kvm_vgic_global_state.maint_irq);
851 return 0;
852 }
853