1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2015, 2016 ARM Ltd.
4 */
5 #ifndef __KVM_ARM_VGIC_H
6 #define __KVM_ARM_VGIC_H
7
8 #include <linux/bits.h>
9 #include <linux/kvm.h>
10 #include <linux/irqreturn.h>
11 #include <linux/mutex.h>
12 #include <linux/refcount.h>
13 #include <linux/spinlock.h>
14 #include <linux/static_key.h>
15 #include <linux/types.h>
16 #include <linux/xarray.h>
17 #include <kvm/iodev.h>
18 #include <linux/list.h>
19 #include <linux/jump_label.h>
20
21 #include <linux/irqchip/arm-gic-v4.h>
22
23 #define VGIC_V3_MAX_CPUS 512
24 #define VGIC_V2_MAX_CPUS 8
25 #define VGIC_NR_IRQS_LEGACY 256
26 #define VGIC_NR_SGIS 16
27 #define VGIC_NR_PPIS 16
28 #define VGIC_NR_PRIVATE_IRQS (VGIC_NR_SGIS + VGIC_NR_PPIS)
29 #define VGIC_MAX_SPI 1019
30 #define VGIC_MAX_RESERVED 1023
31 #define VGIC_MIN_LPI 8192
32 #define KVM_IRQCHIP_NUM_PINS (1020 - 32)
33
34 #define irq_is_ppi(irq) ((irq) >= VGIC_NR_SGIS && (irq) < VGIC_NR_PRIVATE_IRQS)
35 #define irq_is_spi(irq) ((irq) >= VGIC_NR_PRIVATE_IRQS && \
36 (irq) <= VGIC_MAX_SPI)
37
38 enum vgic_type {
39 VGIC_V2, /* Good ol' GICv2 */
40 VGIC_V3, /* New fancy GICv3 */
41 VGIC_V5, /* Newer, fancier GICv5 */
42 };
43
44 /* same for all guests, as depending only on the _host's_ GIC model */
45 struct vgic_global {
46 /* type of the host GIC */
47 enum vgic_type type;
48
49 /* Physical address of vgic virtual cpu interface */
50 phys_addr_t vcpu_base;
51
52 /* GICV mapping, kernel VA */
53 void __iomem *vcpu_base_va;
54 /* GICV mapping, HYP VA */
55 void __iomem *vcpu_hyp_va;
56
57 /* virtual control interface mapping, kernel VA */
58 void __iomem *vctrl_base;
59 /* virtual control interface mapping, HYP VA */
60 void __iomem *vctrl_hyp;
61
62 /* Physical CPU interface, kernel VA */
63 void __iomem *gicc_base;
64
65 /* Number of implemented list registers */
66 int nr_lr;
67
68 /* Maintenance IRQ number */
69 unsigned int maint_irq;
70
71 /* maximum number of VCPUs allowed (GICv2 limits us to 8) */
72 int max_gic_vcpus;
73
74 /* Only needed for the legacy KVM_CREATE_IRQCHIP */
75 bool can_emulate_gicv2;
76
77 /* Hardware has GICv4? */
78 bool has_gicv4;
79 bool has_gicv4_1;
80
81 /* Pseudo GICv3 from outer space */
82 bool no_hw_deactivation;
83
84 /* GICv3 system register CPU interface */
85 struct static_key_false gicv3_cpuif;
86
87 /* GICv3 compat mode on a GICv5 host */
88 bool has_gcie_v3_compat;
89
90 u32 ich_vtr_el2;
91 };
92
93 extern struct vgic_global kvm_vgic_global_state;
94
95 #define VGIC_V2_MAX_LRS (1 << 6)
96 #define VGIC_V3_MAX_LRS 16
97 #define VGIC_V3_LR_INDEX(lr) (VGIC_V3_MAX_LRS - 1 - lr)
98
99 enum vgic_irq_config {
100 VGIC_CONFIG_EDGE = 0,
101 VGIC_CONFIG_LEVEL
102 };
103
104 /*
105 * Per-irq ops overriding some common behavious.
106 *
107 * Always called in non-preemptible section and the functions can use
108 * kvm_arm_get_running_vcpu() to get the vcpu pointer for private IRQs.
109 */
110 struct irq_ops {
111 /* Per interrupt flags for special-cased interrupts */
112 unsigned long flags;
113
114 #define VGIC_IRQ_SW_RESAMPLE BIT(0) /* Clear the active state for resampling */
115
116 /*
117 * Callback function pointer to in-kernel devices that can tell us the
118 * state of the input level of mapped level-triggered IRQ faster than
119 * peaking into the physical GIC.
120 */
121 bool (*get_input_level)(int vintid);
122 };
123
124 struct vgic_irq {
125 raw_spinlock_t irq_lock; /* Protects the content of the struct */
126 u32 intid; /* Guest visible INTID */
127 struct rcu_head rcu;
128 struct list_head ap_list;
129
130 struct kvm_vcpu *vcpu; /* SGIs and PPIs: The VCPU
131 * SPIs and LPIs: The VCPU whose ap_list
132 * this is queued on.
133 */
134
135 struct kvm_vcpu *target_vcpu; /* The VCPU that this interrupt should
136 * be sent to, as a result of the
137 * targets reg (v2) or the
138 * affinity reg (v3).
139 */
140
141 bool pending_release:1; /* Used for LPIs only, unreferenced IRQ
142 * pending a release */
143
144 bool pending_latch:1; /* The pending latch state used to calculate
145 * the pending state for both level
146 * and edge triggered IRQs. */
147 enum vgic_irq_config config:1; /* Level or edge */
148 bool line_level:1; /* Level only */
149 bool enabled:1;
150 bool active:1;
151 bool hw:1; /* Tied to HW IRQ */
152 bool on_lr:1; /* Present in a CPU LR */
153 refcount_t refcount; /* Used for LPIs */
154 u32 hwintid; /* HW INTID number */
155 unsigned int host_irq; /* linux irq corresponding to hwintid */
156 union {
157 u8 targets; /* GICv2 target VCPUs mask */
158 u32 mpidr; /* GICv3 target VCPU */
159 };
160 u8 source; /* GICv2 SGIs only */
161 u8 active_source; /* GICv2 SGIs only */
162 u8 priority;
163 u8 group; /* 0 == group 0, 1 == group 1 */
164
165 struct irq_ops *ops;
166
167 void *owner; /* Opaque pointer to reserve an interrupt
168 for in-kernel devices. */
169 };
170
vgic_irq_needs_resampling(struct vgic_irq * irq)171 static inline bool vgic_irq_needs_resampling(struct vgic_irq *irq)
172 {
173 return irq->ops && (irq->ops->flags & VGIC_IRQ_SW_RESAMPLE);
174 }
175
176 struct vgic_register_region;
177 struct vgic_its;
178
179 enum iodev_type {
180 IODEV_CPUIF,
181 IODEV_DIST,
182 IODEV_REDIST,
183 IODEV_ITS
184 };
185
186 struct vgic_io_device {
187 gpa_t base_addr;
188 union {
189 struct kvm_vcpu *redist_vcpu;
190 struct vgic_its *its;
191 };
192 const struct vgic_register_region *regions;
193 enum iodev_type iodev_type;
194 int nr_regions;
195 struct kvm_io_device dev;
196 };
197
198 struct vgic_its {
199 /* The base address of the ITS control register frame */
200 gpa_t vgic_its_base;
201
202 bool enabled;
203 struct vgic_io_device iodev;
204 struct kvm_device *dev;
205
206 /* These registers correspond to GITS_BASER{0,1} */
207 u64 baser_device_table;
208 u64 baser_coll_table;
209
210 /* Protects the command queue */
211 struct mutex cmd_lock;
212 u64 cbaser;
213 u32 creadr;
214 u32 cwriter;
215
216 /* migration ABI revision in use */
217 u32 abi_rev;
218
219 /* Protects the device and collection lists */
220 struct mutex its_lock;
221 struct list_head device_list;
222 struct list_head collection_list;
223
224 /*
225 * Caches the (device_id, event_id) -> vgic_irq translation for
226 * LPIs that are mapped and enabled.
227 */
228 struct xarray translation_cache;
229 };
230
231 struct vgic_state_iter;
232
233 struct vgic_redist_region {
234 u32 index;
235 gpa_t base;
236 u32 count; /* number of redistributors or 0 if single region */
237 u32 free_index; /* index of the next free redistributor */
238 struct list_head list;
239 };
240
241 struct vgic_dist {
242 bool in_kernel;
243 bool ready;
244 bool initialized;
245
246 /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */
247 u32 vgic_model;
248
249 /* Implementation revision as reported in the GICD_IIDR */
250 u32 implementation_rev;
251 #define KVM_VGIC_IMP_REV_2 2 /* GICv2 restorable groups */
252 #define KVM_VGIC_IMP_REV_3 3 /* GICv3 GICR_CTLR.{IW,CES,RWP} */
253 #define KVM_VGIC_IMP_REV_LATEST KVM_VGIC_IMP_REV_3
254
255 /* Userspace can write to GICv2 IGROUPR */
256 bool v2_groups_user_writable;
257
258 /* Do injected MSIs require an additional device ID? */
259 bool msis_require_devid;
260
261 int nr_spis;
262
263 /* The GIC maintenance IRQ for nested hypervisors. */
264 u32 mi_intid;
265
266 /* Track the number of in-flight active SPIs */
267 atomic_t active_spis;
268
269 /* base addresses in guest physical address space: */
270 gpa_t vgic_dist_base; /* distributor */
271 union {
272 /* either a GICv2 CPU interface */
273 gpa_t vgic_cpu_base;
274 /* or a number of GICv3 redistributor regions */
275 struct list_head rd_regions;
276 };
277
278 /* distributor enabled */
279 bool enabled;
280
281 /* Supports SGIs without active state */
282 bool nassgicap;
283
284 /* Wants SGIs without active state */
285 bool nassgireq;
286
287 struct vgic_irq *spis;
288
289 struct vgic_io_device dist_iodev;
290 struct vgic_io_device cpuif_iodev;
291
292 bool has_its;
293 bool table_write_in_progress;
294
295 /*
296 * Contains the attributes and gpa of the LPI configuration table.
297 * Since we report GICR_TYPER.CommonLPIAff as 0b00, we can share
298 * one address across all redistributors.
299 * GICv3 spec: IHI 0069E 6.1.1 "LPI Configuration tables"
300 */
301 u64 propbaser;
302
303 #define LPI_XA_MARK_DEBUG_ITER XA_MARK_0
304 struct xarray lpi_xa;
305
306 /* used by vgic-debug */
307 struct vgic_state_iter *iter;
308
309 /*
310 * GICv4 ITS per-VM data, containing the IRQ domain, the VPE
311 * array, the property table pointer as well as allocation
312 * data. This essentially ties the Linux IRQ core and ITS
313 * together, and avoids leaking KVM's data structures anywhere
314 * else.
315 */
316 struct its_vm its_vm;
317 };
318
319 struct vgic_v2_cpu_if {
320 u32 vgic_hcr;
321 u32 vgic_vmcr;
322 u32 vgic_apr;
323 u32 vgic_lr[VGIC_V2_MAX_LRS];
324
325 unsigned int used_lrs;
326 };
327
328 struct vgic_v3_cpu_if {
329 u32 vgic_hcr;
330 u32 vgic_vmcr;
331 u32 vgic_sre; /* Restored only, change ignored */
332 u32 vgic_ap0r[4];
333 u32 vgic_ap1r[4];
334 u64 vgic_lr[VGIC_V3_MAX_LRS];
335
336 /*
337 * GICv4 ITS per-VPE data, containing the doorbell IRQ, the
338 * pending table pointer, the its_vm pointer and a few other
339 * HW specific things. As for the its_vm structure, this is
340 * linking the Linux IRQ subsystem and the ITS together.
341 */
342 struct its_vpe its_vpe;
343
344 unsigned int used_lrs;
345 };
346
347 struct vgic_cpu {
348 /* CPU vif control registers for world switch */
349 union {
350 struct vgic_v2_cpu_if vgic_v2;
351 struct vgic_v3_cpu_if vgic_v3;
352 };
353
354 struct vgic_irq *private_irqs;
355
356 raw_spinlock_t ap_list_lock; /* Protects the ap_list */
357
358 /*
359 * List of IRQs that this VCPU should consider because they are either
360 * Active or Pending (hence the name; AP list), or because they recently
361 * were one of the two and need to be migrated off this list to another
362 * VCPU.
363 */
364 struct list_head ap_list_head;
365
366 /*
367 * Members below are used with GICv3 emulation only and represent
368 * parts of the redistributor.
369 */
370 struct vgic_io_device rd_iodev;
371 struct vgic_redist_region *rdreg;
372 u32 rdreg_index;
373 atomic_t syncr_busy;
374
375 /* Contains the attributes and gpa of the LPI pending tables. */
376 u64 pendbaser;
377 /* GICR_CTLR.{ENABLE_LPIS,RWP} */
378 atomic_t ctlr;
379
380 /* Cache guest priority bits */
381 u32 num_pri_bits;
382
383 /* Cache guest interrupt ID bits */
384 u32 num_id_bits;
385 };
386
387 extern struct static_key_false vgic_v2_cpuif_trap;
388 extern struct static_key_false vgic_v3_cpuif_trap;
389 extern struct static_key_false vgic_v3_has_v2_compat;
390
391 int kvm_set_legacy_vgic_v2_addr(struct kvm *kvm, struct kvm_arm_device_addr *dev_addr);
392 void kvm_vgic_early_init(struct kvm *kvm);
393 int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu);
394 int kvm_vgic_vcpu_nv_init(struct kvm_vcpu *vcpu);
395 int kvm_vgic_create(struct kvm *kvm, u32 type);
396 void kvm_vgic_destroy(struct kvm *kvm);
397 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu);
398 int kvm_vgic_map_resources(struct kvm *kvm);
399 int kvm_vgic_hyp_init(void);
400 void kvm_vgic_init_cpu_hardware(void);
401
402 int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
403 unsigned int intid, bool level, void *owner);
404 int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq,
405 u32 vintid, struct irq_ops *ops);
406 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid);
407 int kvm_vgic_get_map(struct kvm_vcpu *vcpu, unsigned int vintid);
408 bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid);
409
410 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu);
411
412 void kvm_vgic_load(struct kvm_vcpu *vcpu);
413 void kvm_vgic_put(struct kvm_vcpu *vcpu);
414
415 u16 vgic_v3_get_eisr(struct kvm_vcpu *vcpu);
416 u16 vgic_v3_get_elrsr(struct kvm_vcpu *vcpu);
417 u64 vgic_v3_get_misr(struct kvm_vcpu *vcpu);
418
419 #define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel))
420 #define vgic_initialized(k) ((k)->arch.vgic.initialized)
421 #define vgic_valid_spi(k, i) (((i) >= VGIC_NR_PRIVATE_IRQS) && \
422 ((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS))
423
424 bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu);
425 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu);
426 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu);
427 void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid);
428 void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu);
429
430 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1);
431
432 /**
433 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW
434 *
435 * The host's GIC naturally limits the maximum amount of VCPUs a guest
436 * can use.
437 */
kvm_vgic_get_max_vcpus(void)438 static inline int kvm_vgic_get_max_vcpus(void)
439 {
440 return kvm_vgic_global_state.max_gic_vcpus;
441 }
442
443 /**
444 * kvm_vgic_setup_default_irq_routing:
445 * Setup a default flat gsi routing table mapping all SPIs
446 */
447 int kvm_vgic_setup_default_irq_routing(struct kvm *kvm);
448
449 int kvm_vgic_set_owner(struct kvm_vcpu *vcpu, unsigned int intid, void *owner);
450
451 struct kvm_kernel_irq_routing_entry;
452
453 int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int irq,
454 struct kvm_kernel_irq_routing_entry *irq_entry);
455
456 void kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int host_irq);
457
458 int vgic_v4_load(struct kvm_vcpu *vcpu);
459 void vgic_v4_commit(struct kvm_vcpu *vcpu);
460 int vgic_v4_put(struct kvm_vcpu *vcpu);
461
462 bool vgic_state_is_nested(struct kvm_vcpu *vcpu);
463
464 /* CPU HP callbacks */
465 void kvm_vgic_cpu_up(void);
466 void kvm_vgic_cpu_down(void);
467
468 #endif /* __KVM_ARM_VGIC_H */
469