1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2015, 2016 ARM Ltd. 4 */ 5 #ifndef __KVM_ARM_VGIC_H 6 #define __KVM_ARM_VGIC_H 7 8 #include <linux/bits.h> 9 #include <linux/kvm.h> 10 #include <linux/irqreturn.h> 11 #include <linux/mutex.h> 12 #include <linux/refcount.h> 13 #include <linux/spinlock.h> 14 #include <linux/static_key.h> 15 #include <linux/types.h> 16 #include <linux/xarray.h> 17 #include <kvm/iodev.h> 18 #include <linux/list.h> 19 #include <linux/jump_label.h> 20 21 #include <linux/irqchip/arm-gic-v4.h> 22 23 #define VGIC_V3_MAX_CPUS 512 24 #define VGIC_V2_MAX_CPUS 8 25 #define VGIC_NR_IRQS_LEGACY 256 26 #define VGIC_NR_SGIS 16 27 #define VGIC_NR_PPIS 16 28 #define VGIC_NR_PRIVATE_IRQS (VGIC_NR_SGIS + VGIC_NR_PPIS) 29 #define VGIC_MAX_SPI 1019 30 #define VGIC_MAX_RESERVED 1023 31 #define VGIC_MIN_LPI 8192 32 #define KVM_IRQCHIP_NUM_PINS (1020 - 32) 33 34 #define irq_is_ppi(irq) ((irq) >= VGIC_NR_SGIS && (irq) < VGIC_NR_PRIVATE_IRQS) 35 #define irq_is_spi(irq) ((irq) >= VGIC_NR_PRIVATE_IRQS && \ 36 (irq) <= VGIC_MAX_SPI) 37 38 enum vgic_type { 39 VGIC_V2, /* Good ol' GICv2 */ 40 VGIC_V3, /* New fancy GICv3 */ 41 VGIC_V5, /* Newer, fancier GICv5 */ 42 }; 43 44 /* same for all guests, as depending only on the _host's_ GIC model */ 45 struct vgic_global { 46 /* type of the host GIC */ 47 enum vgic_type type; 48 49 /* Physical address of vgic virtual cpu interface */ 50 phys_addr_t vcpu_base; 51 52 /* GICV mapping, kernel VA */ 53 void __iomem *vcpu_base_va; 54 /* GICV mapping, HYP VA */ 55 void __iomem *vcpu_hyp_va; 56 57 /* virtual control interface mapping, kernel VA */ 58 void __iomem *vctrl_base; 59 /* virtual control interface mapping, HYP VA */ 60 void __iomem *vctrl_hyp; 61 62 /* Physical CPU interface, kernel VA */ 63 void __iomem *gicc_base; 64 65 /* Number of implemented list registers */ 66 int nr_lr; 67 68 /* Maintenance IRQ number */ 69 unsigned int maint_irq; 70 71 /* maximum number of VCPUs allowed (GICv2 limits us to 8) */ 72 int max_gic_vcpus; 73 74 /* Only needed for the legacy KVM_CREATE_IRQCHIP */ 75 bool can_emulate_gicv2; 76 77 /* Hardware has GICv4? */ 78 bool has_gicv4; 79 bool has_gicv4_1; 80 81 /* Pseudo GICv3 from outer space */ 82 bool no_hw_deactivation; 83 84 /* GICv3 system register CPU interface */ 85 struct static_key_false gicv3_cpuif; 86 87 /* GICv3 compat mode on a GICv5 host */ 88 bool has_gcie_v3_compat; 89 90 u32 ich_vtr_el2; 91 }; 92 93 extern struct vgic_global kvm_vgic_global_state; 94 95 #define VGIC_V2_MAX_LRS (1 << 6) 96 #define VGIC_V3_MAX_LRS 16 97 #define VGIC_V3_LR_INDEX(lr) (VGIC_V3_MAX_LRS - 1 - lr) 98 99 enum vgic_irq_config { 100 VGIC_CONFIG_EDGE = 0, 101 VGIC_CONFIG_LEVEL 102 }; 103 104 /* 105 * Per-irq ops overriding some common behavious. 106 * 107 * Always called in non-preemptible section and the functions can use 108 * kvm_arm_get_running_vcpu() to get the vcpu pointer for private IRQs. 109 */ 110 struct irq_ops { 111 /* Per interrupt flags for special-cased interrupts */ 112 unsigned long flags; 113 114 #define VGIC_IRQ_SW_RESAMPLE BIT(0) /* Clear the active state for resampling */ 115 116 /* 117 * Callback function pointer to in-kernel devices that can tell us the 118 * state of the input level of mapped level-triggered IRQ faster than 119 * peaking into the physical GIC. 120 */ 121 bool (*get_input_level)(int vintid); 122 }; 123 124 struct vgic_irq { 125 raw_spinlock_t irq_lock; /* Protects the content of the struct */ 126 u32 intid; /* Guest visible INTID */ 127 struct rcu_head rcu; 128 struct list_head ap_list; 129 130 struct kvm_vcpu *vcpu; /* SGIs and PPIs: The VCPU 131 * SPIs and LPIs: The VCPU whose ap_list 132 * this is queued on. 133 */ 134 135 struct kvm_vcpu *target_vcpu; /* The VCPU that this interrupt should 136 * be sent to, as a result of the 137 * targets reg (v2) or the 138 * affinity reg (v3). 139 */ 140 141 bool pending_release:1; /* Used for LPIs only, unreferenced IRQ 142 * pending a release */ 143 144 bool pending_latch:1; /* The pending latch state used to calculate 145 * the pending state for both level 146 * and edge triggered IRQs. */ 147 enum vgic_irq_config config:1; /* Level or edge */ 148 bool line_level:1; /* Level only */ 149 bool enabled:1; 150 bool active:1; 151 bool hw:1; /* Tied to HW IRQ */ 152 bool on_lr:1; /* Present in a CPU LR */ 153 refcount_t refcount; /* Used for LPIs */ 154 u32 hwintid; /* HW INTID number */ 155 unsigned int host_irq; /* linux irq corresponding to hwintid */ 156 union { 157 u8 targets; /* GICv2 target VCPUs mask */ 158 u32 mpidr; /* GICv3 target VCPU */ 159 }; 160 u8 source; /* GICv2 SGIs only */ 161 u8 active_source; /* GICv2 SGIs only */ 162 u8 priority; 163 u8 group; /* 0 == group 0, 1 == group 1 */ 164 165 struct irq_ops *ops; 166 167 void *owner; /* Opaque pointer to reserve an interrupt 168 for in-kernel devices. */ 169 }; 170 171 static inline bool vgic_irq_needs_resampling(struct vgic_irq *irq) 172 { 173 return irq->ops && (irq->ops->flags & VGIC_IRQ_SW_RESAMPLE); 174 } 175 176 struct vgic_register_region; 177 struct vgic_its; 178 179 enum iodev_type { 180 IODEV_CPUIF, 181 IODEV_DIST, 182 IODEV_REDIST, 183 IODEV_ITS 184 }; 185 186 struct vgic_io_device { 187 gpa_t base_addr; 188 union { 189 struct kvm_vcpu *redist_vcpu; 190 struct vgic_its *its; 191 }; 192 const struct vgic_register_region *regions; 193 enum iodev_type iodev_type; 194 int nr_regions; 195 struct kvm_io_device dev; 196 }; 197 198 struct vgic_its { 199 /* The base address of the ITS control register frame */ 200 gpa_t vgic_its_base; 201 202 bool enabled; 203 struct vgic_io_device iodev; 204 struct kvm_device *dev; 205 206 /* These registers correspond to GITS_BASER{0,1} */ 207 u64 baser_device_table; 208 u64 baser_coll_table; 209 210 /* Protects the command queue */ 211 struct mutex cmd_lock; 212 u64 cbaser; 213 u32 creadr; 214 u32 cwriter; 215 216 /* migration ABI revision in use */ 217 u32 abi_rev; 218 219 /* Protects the device and collection lists */ 220 struct mutex its_lock; 221 struct list_head device_list; 222 struct list_head collection_list; 223 224 /* 225 * Caches the (device_id, event_id) -> vgic_irq translation for 226 * LPIs that are mapped and enabled. 227 */ 228 struct xarray translation_cache; 229 }; 230 231 struct vgic_state_iter; 232 233 struct vgic_redist_region { 234 u32 index; 235 gpa_t base; 236 u32 count; /* number of redistributors or 0 if single region */ 237 u32 free_index; /* index of the next free redistributor */ 238 struct list_head list; 239 }; 240 241 struct vgic_dist { 242 bool in_kernel; 243 bool ready; 244 bool initialized; 245 246 /* vGIC model the kernel emulates for the guest (GICv2 or GICv3) */ 247 u32 vgic_model; 248 249 /* Implementation revision as reported in the GICD_IIDR */ 250 u32 implementation_rev; 251 #define KVM_VGIC_IMP_REV_2 2 /* GICv2 restorable groups */ 252 #define KVM_VGIC_IMP_REV_3 3 /* GICv3 GICR_CTLR.{IW,CES,RWP} */ 253 #define KVM_VGIC_IMP_REV_LATEST KVM_VGIC_IMP_REV_3 254 255 /* Userspace can write to GICv2 IGROUPR */ 256 bool v2_groups_user_writable; 257 258 /* Do injected MSIs require an additional device ID? */ 259 bool msis_require_devid; 260 261 int nr_spis; 262 263 /* The GIC maintenance IRQ for nested hypervisors. */ 264 u32 mi_intid; 265 266 /* Track the number of in-flight active SPIs */ 267 atomic_t active_spis; 268 269 /* base addresses in guest physical address space: */ 270 gpa_t vgic_dist_base; /* distributor */ 271 union { 272 /* either a GICv2 CPU interface */ 273 gpa_t vgic_cpu_base; 274 /* or a number of GICv3 redistributor regions */ 275 struct list_head rd_regions; 276 }; 277 278 /* distributor enabled */ 279 bool enabled; 280 281 /* Supports SGIs without active state */ 282 bool nassgicap; 283 284 /* Wants SGIs without active state */ 285 bool nassgireq; 286 287 struct vgic_irq *spis; 288 289 struct vgic_io_device dist_iodev; 290 struct vgic_io_device cpuif_iodev; 291 292 bool has_its; 293 bool table_write_in_progress; 294 295 /* 296 * Contains the attributes and gpa of the LPI configuration table. 297 * Since we report GICR_TYPER.CommonLPIAff as 0b00, we can share 298 * one address across all redistributors. 299 * GICv3 spec: IHI 0069E 6.1.1 "LPI Configuration tables" 300 */ 301 u64 propbaser; 302 303 struct xarray lpi_xa; 304 305 /* 306 * GICv4 ITS per-VM data, containing the IRQ domain, the VPE 307 * array, the property table pointer as well as allocation 308 * data. This essentially ties the Linux IRQ core and ITS 309 * together, and avoids leaking KVM's data structures anywhere 310 * else. 311 */ 312 struct its_vm its_vm; 313 }; 314 315 struct vgic_v2_cpu_if { 316 u32 vgic_hcr; 317 u32 vgic_vmcr; 318 u32 vgic_apr; 319 u32 vgic_lr[VGIC_V2_MAX_LRS]; 320 321 unsigned int used_lrs; 322 }; 323 324 struct vgic_v3_cpu_if { 325 u32 vgic_hcr; 326 u32 vgic_vmcr; 327 u32 vgic_sre; /* Restored only, change ignored */ 328 u32 vgic_ap0r[4]; 329 u32 vgic_ap1r[4]; 330 u64 vgic_lr[VGIC_V3_MAX_LRS]; 331 332 /* 333 * GICv4 ITS per-VPE data, containing the doorbell IRQ, the 334 * pending table pointer, the its_vm pointer and a few other 335 * HW specific things. As for the its_vm structure, this is 336 * linking the Linux IRQ subsystem and the ITS together. 337 */ 338 struct its_vpe its_vpe; 339 340 unsigned int used_lrs; 341 }; 342 343 struct vgic_cpu { 344 /* CPU vif control registers for world switch */ 345 union { 346 struct vgic_v2_cpu_if vgic_v2; 347 struct vgic_v3_cpu_if vgic_v3; 348 }; 349 350 struct vgic_irq *private_irqs; 351 352 raw_spinlock_t ap_list_lock; /* Protects the ap_list */ 353 354 /* 355 * List of IRQs that this VCPU should consider because they are either 356 * Active or Pending (hence the name; AP list), or because they recently 357 * were one of the two and need to be migrated off this list to another 358 * VCPU. 359 */ 360 struct list_head ap_list_head; 361 362 /* 363 * Members below are used with GICv3 emulation only and represent 364 * parts of the redistributor. 365 */ 366 struct vgic_io_device rd_iodev; 367 struct vgic_redist_region *rdreg; 368 u32 rdreg_index; 369 atomic_t syncr_busy; 370 371 /* Contains the attributes and gpa of the LPI pending tables. */ 372 u64 pendbaser; 373 /* GICR_CTLR.{ENABLE_LPIS,RWP} */ 374 atomic_t ctlr; 375 376 /* Cache guest priority bits */ 377 u32 num_pri_bits; 378 379 /* Cache guest interrupt ID bits */ 380 u32 num_id_bits; 381 }; 382 383 extern struct static_key_false vgic_v2_cpuif_trap; 384 extern struct static_key_false vgic_v3_cpuif_trap; 385 extern struct static_key_false vgic_v3_has_v2_compat; 386 387 int kvm_set_legacy_vgic_v2_addr(struct kvm *kvm, struct kvm_arm_device_addr *dev_addr); 388 void kvm_vgic_early_init(struct kvm *kvm); 389 int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu); 390 int kvm_vgic_vcpu_nv_init(struct kvm_vcpu *vcpu); 391 int kvm_vgic_create(struct kvm *kvm, u32 type); 392 void kvm_vgic_destroy(struct kvm *kvm); 393 void kvm_vgic_vcpu_destroy(struct kvm_vcpu *vcpu); 394 int kvm_vgic_map_resources(struct kvm *kvm); 395 int kvm_vgic_hyp_init(void); 396 void kvm_vgic_init_cpu_hardware(void); 397 398 int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu, 399 unsigned int intid, bool level, void *owner); 400 int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq, 401 u32 vintid, struct irq_ops *ops); 402 int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid); 403 int kvm_vgic_get_map(struct kvm_vcpu *vcpu, unsigned int vintid); 404 bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid); 405 406 int kvm_vgic_vcpu_pending_irq(struct kvm_vcpu *vcpu); 407 408 void kvm_vgic_load(struct kvm_vcpu *vcpu); 409 void kvm_vgic_put(struct kvm_vcpu *vcpu); 410 411 u16 vgic_v3_get_eisr(struct kvm_vcpu *vcpu); 412 u16 vgic_v3_get_elrsr(struct kvm_vcpu *vcpu); 413 u64 vgic_v3_get_misr(struct kvm_vcpu *vcpu); 414 415 #define irqchip_in_kernel(k) (!!((k)->arch.vgic.in_kernel)) 416 #define vgic_initialized(k) ((k)->arch.vgic.initialized) 417 #define vgic_valid_spi(k, i) (((i) >= VGIC_NR_PRIVATE_IRQS) && \ 418 ((i) < (k)->arch.vgic.nr_spis + VGIC_NR_PRIVATE_IRQS)) 419 420 bool kvm_vcpu_has_pending_irqs(struct kvm_vcpu *vcpu); 421 void kvm_vgic_sync_hwstate(struct kvm_vcpu *vcpu); 422 void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu); 423 void kvm_vgic_reset_mapped_irq(struct kvm_vcpu *vcpu, u32 vintid); 424 void kvm_vgic_process_async_update(struct kvm_vcpu *vcpu); 425 426 void vgic_v3_dispatch_sgi(struct kvm_vcpu *vcpu, u64 reg, bool allow_group1); 427 428 /** 429 * kvm_vgic_get_max_vcpus - Get the maximum number of VCPUs allowed by HW 430 * 431 * The host's GIC naturally limits the maximum amount of VCPUs a guest 432 * can use. 433 */ 434 static inline int kvm_vgic_get_max_vcpus(void) 435 { 436 return kvm_vgic_global_state.max_gic_vcpus; 437 } 438 439 /** 440 * kvm_vgic_setup_default_irq_routing: 441 * Setup a default flat gsi routing table mapping all SPIs 442 */ 443 int kvm_vgic_setup_default_irq_routing(struct kvm *kvm); 444 445 int kvm_vgic_set_owner(struct kvm_vcpu *vcpu, unsigned int intid, void *owner); 446 447 struct kvm_kernel_irq_routing_entry; 448 449 int kvm_vgic_v4_set_forwarding(struct kvm *kvm, int irq, 450 struct kvm_kernel_irq_routing_entry *irq_entry); 451 452 void kvm_vgic_v4_unset_forwarding(struct kvm *kvm, int host_irq); 453 454 int vgic_v4_load(struct kvm_vcpu *vcpu); 455 void vgic_v4_commit(struct kvm_vcpu *vcpu); 456 int vgic_v4_put(struct kvm_vcpu *vcpu); 457 458 bool vgic_state_is_nested(struct kvm_vcpu *vcpu); 459 460 /* CPU HP callbacks */ 461 void kvm_vgic_cpu_up(void); 462 void kvm_vgic_cpu_down(void); 463 464 #endif /* __KVM_ARM_VGIC_H */ 465