1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2015, 2016 ARM Ltd. 4 */ 5 #ifndef __KVM_ARM_VGIC_NEW_H__ 6 #define __KVM_ARM_VGIC_NEW_H__ 7 8 #include <linux/irqchip/arm-gic-common.h> 9 #include <asm/kvm_mmu.h> 10 11 #define PRODUCT_ID_KVM 0x4b /* ASCII code K */ 12 #define IMPLEMENTER_ARM 0x43b 13 14 #define VGIC_ADDR_UNDEF (-1) 15 #define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF) 16 17 #define INTERRUPT_ID_BITS_SPIS 10 18 #define INTERRUPT_ID_BITS_ITS 16 19 #define VGIC_LPI_MAX_INTID ((1 << INTERRUPT_ID_BITS_ITS) - 1) 20 #define VGIC_PRI_BITS 5 21 22 #define vgic_irq_is_sgi(intid) ((intid) < VGIC_NR_SGIS) 23 24 #define VGIC_AFFINITY_0_SHIFT 0 25 #define VGIC_AFFINITY_0_MASK (0xffUL << VGIC_AFFINITY_0_SHIFT) 26 #define VGIC_AFFINITY_1_SHIFT 8 27 #define VGIC_AFFINITY_1_MASK (0xffUL << VGIC_AFFINITY_1_SHIFT) 28 #define VGIC_AFFINITY_2_SHIFT 16 29 #define VGIC_AFFINITY_2_MASK (0xffUL << VGIC_AFFINITY_2_SHIFT) 30 #define VGIC_AFFINITY_3_SHIFT 24 31 #define VGIC_AFFINITY_3_MASK (0xffUL << VGIC_AFFINITY_3_SHIFT) 32 33 #define VGIC_AFFINITY_LEVEL(reg, level) \ 34 ((((reg) & VGIC_AFFINITY_## level ##_MASK) \ 35 >> VGIC_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level)) 36 37 /* 38 * The Userspace encodes the affinity differently from the MPIDR, 39 * Below macro converts vgic userspace format to MPIDR reg format. 40 */ 41 #define VGIC_TO_MPIDR(val) (VGIC_AFFINITY_LEVEL(val, 0) | \ 42 VGIC_AFFINITY_LEVEL(val, 1) | \ 43 VGIC_AFFINITY_LEVEL(val, 2) | \ 44 VGIC_AFFINITY_LEVEL(val, 3)) 45 46 /* 47 * As per Documentation/virt/kvm/devices/arm-vgic-v3.rst, 48 * below macros are defined for CPUREG encoding. 49 */ 50 #define KVM_REG_ARM_VGIC_SYSREG_OP0_MASK 0x000000000000c000 51 #define KVM_REG_ARM_VGIC_SYSREG_OP0_SHIFT 14 52 #define KVM_REG_ARM_VGIC_SYSREG_OP1_MASK 0x0000000000003800 53 #define KVM_REG_ARM_VGIC_SYSREG_OP1_SHIFT 11 54 #define KVM_REG_ARM_VGIC_SYSREG_CRN_MASK 0x0000000000000780 55 #define KVM_REG_ARM_VGIC_SYSREG_CRN_SHIFT 7 56 #define KVM_REG_ARM_VGIC_SYSREG_CRM_MASK 0x0000000000000078 57 #define KVM_REG_ARM_VGIC_SYSREG_CRM_SHIFT 3 58 #define KVM_REG_ARM_VGIC_SYSREG_OP2_MASK 0x0000000000000007 59 #define KVM_REG_ARM_VGIC_SYSREG_OP2_SHIFT 0 60 61 #define KVM_DEV_ARM_VGIC_SYSREG_MASK (KVM_REG_ARM_VGIC_SYSREG_OP0_MASK | \ 62 KVM_REG_ARM_VGIC_SYSREG_OP1_MASK | \ 63 KVM_REG_ARM_VGIC_SYSREG_CRN_MASK | \ 64 KVM_REG_ARM_VGIC_SYSREG_CRM_MASK | \ 65 KVM_REG_ARM_VGIC_SYSREG_OP2_MASK) 66 67 #define KVM_ICC_SRE_EL2 (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE | \ 68 ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB) 69 #define KVM_ICH_VTR_EL2_RES0 (ICH_VTR_EL2_DVIM | \ 70 ICH_VTR_EL2_A3V | \ 71 ICH_VTR_EL2_IDbits) 72 #define KVM_ICH_VTR_EL2_RES1 ICH_VTR_EL2_nV4 73 74 static inline u64 kvm_get_guest_vtr_el2(void) 75 { 76 u64 vtr; 77 78 vtr = kvm_vgic_global_state.ich_vtr_el2; 79 vtr &= ~KVM_ICH_VTR_EL2_RES0; 80 vtr |= KVM_ICH_VTR_EL2_RES1; 81 82 return vtr; 83 } 84 85 /* 86 * As per Documentation/virt/kvm/devices/arm-vgic-its.rst, 87 * below macros are defined for ITS table entry encoding. 88 */ 89 #define KVM_ITS_CTE_VALID_SHIFT 63 90 #define KVM_ITS_CTE_VALID_MASK BIT_ULL(63) 91 #define KVM_ITS_CTE_RDBASE_SHIFT 16 92 #define KVM_ITS_CTE_ICID_MASK GENMASK_ULL(15, 0) 93 #define KVM_ITS_ITE_NEXT_SHIFT 48 94 #define KVM_ITS_ITE_PINTID_SHIFT 16 95 #define KVM_ITS_ITE_PINTID_MASK GENMASK_ULL(47, 16) 96 #define KVM_ITS_ITE_ICID_MASK GENMASK_ULL(15, 0) 97 #define KVM_ITS_DTE_VALID_SHIFT 63 98 #define KVM_ITS_DTE_VALID_MASK BIT_ULL(63) 99 #define KVM_ITS_DTE_NEXT_SHIFT 49 100 #define KVM_ITS_DTE_NEXT_MASK GENMASK_ULL(62, 49) 101 #define KVM_ITS_DTE_ITTADDR_SHIFT 5 102 #define KVM_ITS_DTE_ITTADDR_MASK GENMASK_ULL(48, 5) 103 #define KVM_ITS_DTE_SIZE_MASK GENMASK_ULL(4, 0) 104 #define KVM_ITS_L1E_VALID_MASK BIT_ULL(63) 105 /* we only support 64 kB translation table page size */ 106 #define KVM_ITS_L1E_ADDR_MASK GENMASK_ULL(51, 16) 107 108 #define KVM_VGIC_V3_RDIST_INDEX_MASK GENMASK_ULL(11, 0) 109 #define KVM_VGIC_V3_RDIST_FLAGS_MASK GENMASK_ULL(15, 12) 110 #define KVM_VGIC_V3_RDIST_FLAGS_SHIFT 12 111 #define KVM_VGIC_V3_RDIST_BASE_MASK GENMASK_ULL(51, 16) 112 #define KVM_VGIC_V3_RDIST_COUNT_MASK GENMASK_ULL(63, 52) 113 #define KVM_VGIC_V3_RDIST_COUNT_SHIFT 52 114 115 #ifdef CONFIG_DEBUG_SPINLOCK 116 #define DEBUG_SPINLOCK_BUG_ON(p) BUG_ON(p) 117 #else 118 #define DEBUG_SPINLOCK_BUG_ON(p) 119 #endif 120 121 static inline u32 vgic_get_implementation_rev(struct kvm_vcpu *vcpu) 122 { 123 return vcpu->kvm->arch.vgic.implementation_rev; 124 } 125 126 /* Requires the irq_lock to be held by the caller. */ 127 static inline bool irq_is_pending(struct vgic_irq *irq) 128 { 129 if (irq->config == VGIC_CONFIG_EDGE) 130 return irq->pending_latch; 131 else 132 return irq->pending_latch || irq->line_level; 133 } 134 135 static inline bool vgic_irq_is_mapped_level(struct vgic_irq *irq) 136 { 137 return irq->config == VGIC_CONFIG_LEVEL && irq->hw; 138 } 139 140 static inline int vgic_irq_get_lr_count(struct vgic_irq *irq) 141 { 142 /* Account for the active state as an interrupt */ 143 if (vgic_irq_is_sgi(irq->intid) && irq->source) 144 return hweight8(irq->source) + irq->active; 145 146 return irq_is_pending(irq) || irq->active; 147 } 148 149 static inline bool vgic_irq_is_multi_sgi(struct vgic_irq *irq) 150 { 151 return vgic_irq_get_lr_count(irq) > 1; 152 } 153 154 static inline int vgic_write_guest_lock(struct kvm *kvm, gpa_t gpa, 155 const void *data, unsigned long len) 156 { 157 struct vgic_dist *dist = &kvm->arch.vgic; 158 int ret; 159 160 dist->table_write_in_progress = true; 161 ret = kvm_write_guest_lock(kvm, gpa, data, len); 162 dist->table_write_in_progress = false; 163 164 return ret; 165 } 166 167 void kvm_compute_ich_hcr_trap_bits(struct alt_instr *alt, 168 __le32 *origptr, __le32 *updptr, int nr_inst); 169 170 static inline u64 vgic_ich_hcr_trap_bits(void) 171 { 172 u64 hcr; 173 174 /* All the traps are in the bottom 16bits */ 175 asm volatile(ALTERNATIVE_CB("movz %0, #0\n", 176 ARM64_ALWAYS_SYSTEM, 177 kvm_compute_ich_hcr_trap_bits) 178 : "=r" (hcr)); 179 180 return hcr; 181 } 182 183 /* 184 * This struct provides an intermediate representation of the fields contained 185 * in the GICH_VMCR and ICH_VMCR registers, such that code exporting the GIC 186 * state to userspace can generate either GICv2 or GICv3 CPU interface 187 * registers regardless of the hardware backed GIC used. 188 */ 189 struct vgic_vmcr { 190 u32 grpen0; 191 u32 grpen1; 192 193 u32 ackctl; 194 u32 fiqen; 195 u32 cbpr; 196 u32 eoim; 197 198 u32 abpr; 199 u32 bpr; 200 u32 pmr; /* Priority mask field in the GICC_PMR and 201 * ICC_PMR_EL1 priority field format */ 202 }; 203 204 struct vgic_reg_attr { 205 struct kvm_vcpu *vcpu; 206 gpa_t addr; 207 }; 208 209 struct its_device { 210 struct list_head dev_list; 211 212 /* the head for the list of ITTEs */ 213 struct list_head itt_head; 214 u32 num_eventid_bits; 215 gpa_t itt_addr; 216 u32 device_id; 217 }; 218 219 #define COLLECTION_NOT_MAPPED ((u32)~0) 220 221 struct its_collection { 222 struct list_head coll_list; 223 224 u32 collection_id; 225 u32 target_addr; 226 }; 227 228 #define its_is_collection_mapped(coll) ((coll) && \ 229 ((coll)->target_addr != COLLECTION_NOT_MAPPED)) 230 231 struct its_ite { 232 struct list_head ite_list; 233 234 struct vgic_irq *irq; 235 struct its_collection *collection; 236 u32 event_id; 237 }; 238 239 struct ap_list_summary { 240 unsigned int nr_pend; /* purely pending, not active */ 241 unsigned int nr_act; /* active, or active+pending */ 242 unsigned int nr_sgi; /* any SGI */ 243 }; 244 245 #define irqs_outside_lrs(s) \ 246 (((s)->nr_pend + (s)->nr_act) > kvm_vgic_global_state.nr_lr) 247 248 #define irqs_pending_outside_lrs(s) \ 249 ((s)->nr_pend > kvm_vgic_global_state.nr_lr) 250 251 #define irqs_active_outside_lrs(s) \ 252 ((s)->nr_act && irqs_outside_lrs(s)) 253 254 int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, 255 struct vgic_reg_attr *reg_attr); 256 int vgic_v2_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr, 257 struct vgic_reg_attr *reg_attr); 258 const struct vgic_register_region * 259 vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev, 260 gpa_t addr, int len); 261 struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid); 262 struct vgic_irq *vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid); 263 void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq); 264 struct kvm_vcpu *vgic_target_oracle(struct vgic_irq *irq); 265 bool vgic_get_phys_line_level(struct vgic_irq *irq); 266 void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending); 267 void vgic_irq_set_phys_active(struct vgic_irq *irq, bool active); 268 bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq, 269 unsigned long flags) __releases(&irq->irq_lock); 270 void vgic_kick_vcpus(struct kvm *kvm); 271 void vgic_irq_handle_resampling(struct vgic_irq *irq, 272 bool lr_deactivated, bool lr_pending); 273 274 int vgic_check_iorange(struct kvm *kvm, phys_addr_t ioaddr, 275 phys_addr_t addr, phys_addr_t alignment, 276 phys_addr_t size); 277 278 void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu); 279 void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr); 280 void vgic_v2_deactivate(struct kvm_vcpu *vcpu, u32 val); 281 void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr); 282 void vgic_v2_configure_hcr(struct kvm_vcpu *vcpu, struct ap_list_summary *als); 283 int vgic_v2_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr); 284 int vgic_v2_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write, 285 int offset, u32 *val); 286 int vgic_v2_cpuif_uaccess(struct kvm_vcpu *vcpu, bool is_write, 287 int offset, u32 *val); 288 void vgic_v2_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); 289 void vgic_v2_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); 290 void vgic_v2_reset(struct kvm_vcpu *vcpu); 291 int vgic_v2_probe(const struct gic_kvm_info *info); 292 int vgic_v2_map_resources(struct kvm *kvm); 293 int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address, 294 enum vgic_type); 295 296 void vgic_v2_init_lrs(void); 297 void vgic_v2_load(struct kvm_vcpu *vcpu); 298 void vgic_v2_put(struct kvm_vcpu *vcpu); 299 300 void vgic_v2_save_state(struct kvm_vcpu *vcpu); 301 void vgic_v2_restore_state(struct kvm_vcpu *vcpu); 302 303 static inline bool vgic_try_get_irq_ref(struct vgic_irq *irq) 304 { 305 if (!irq) 306 return false; 307 308 if (irq->intid < VGIC_MIN_LPI) 309 return true; 310 311 return refcount_inc_not_zero(&irq->refcount); 312 } 313 314 static inline void vgic_get_irq_ref(struct vgic_irq *irq) 315 { 316 WARN_ON_ONCE(!vgic_try_get_irq_ref(irq)); 317 } 318 319 void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu); 320 void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr); 321 void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr); 322 void vgic_v3_deactivate(struct kvm_vcpu *vcpu, u64 val); 323 void vgic_v3_configure_hcr(struct kvm_vcpu *vcpu, struct ap_list_summary *als); 324 void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); 325 void vgic_v3_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); 326 void vgic_v3_reset(struct kvm_vcpu *vcpu); 327 void vgic_v3_enable_cpuif_traps(void); 328 int vgic_v3_probe(const struct gic_kvm_info *info); 329 int vgic_v3_map_resources(struct kvm *kvm); 330 int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq); 331 int vgic_v3_save_pending_tables(struct kvm *kvm); 332 int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count); 333 int vgic_register_redist_iodev(struct kvm_vcpu *vcpu); 334 void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu); 335 bool vgic_v3_check_base(struct kvm *kvm); 336 337 void vgic_v3_load(struct kvm_vcpu *vcpu); 338 void vgic_v3_put(struct kvm_vcpu *vcpu); 339 340 bool vgic_has_its(struct kvm *kvm); 341 int kvm_vgic_register_its_device(void); 342 void vgic_enable_lpis(struct kvm_vcpu *vcpu); 343 void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu); 344 int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi); 345 int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr); 346 int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write, 347 int offset, u32 *val); 348 int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write, 349 int offset, u32 *val); 350 int vgic_v3_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu, 351 struct kvm_device_attr *attr, bool is_write); 352 int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); 353 const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz); 354 int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write, 355 u32 intid, u32 *val); 356 int kvm_register_vgic_device(unsigned long type); 357 void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); 358 void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr); 359 int vgic_lazy_init(struct kvm *kvm); 360 int vgic_init(struct kvm *kvm); 361 362 void vgic_debug_init(struct kvm *kvm); 363 void vgic_debug_destroy(struct kvm *kvm); 364 365 int vgic_v5_probe(const struct gic_kvm_info *info); 366 367 static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu) 368 { 369 struct vgic_cpu *cpu_if = &vcpu->arch.vgic_cpu; 370 371 /* 372 * num_pri_bits are initialized with HW supported values. 373 * We can rely safely on num_pri_bits even if VM has not 374 * restored ICC_CTLR_EL1 before restoring APnR registers. 375 */ 376 switch (cpu_if->num_pri_bits) { 377 case 7: return 3; 378 case 6: return 1; 379 default: return 0; 380 } 381 } 382 383 static inline bool 384 vgic_v3_redist_region_full(struct vgic_redist_region *region) 385 { 386 if (!region->count) 387 return false; 388 389 return (region->free_index >= region->count); 390 } 391 392 struct vgic_redist_region *vgic_v3_rdist_free_slot(struct list_head *rdregs); 393 394 static inline size_t 395 vgic_v3_rd_region_size(struct kvm *kvm, struct vgic_redist_region *rdreg) 396 { 397 if (!rdreg->count) 398 return atomic_read(&kvm->online_vcpus) * KVM_VGIC_V3_REDIST_SIZE; 399 else 400 return rdreg->count * KVM_VGIC_V3_REDIST_SIZE; 401 } 402 403 struct vgic_redist_region *vgic_v3_rdist_region_from_index(struct kvm *kvm, 404 u32 index); 405 void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdreg); 406 407 bool vgic_v3_rdist_overlap(struct kvm *kvm, gpa_t base, size_t size); 408 409 static inline bool vgic_dist_overlap(struct kvm *kvm, gpa_t base, size_t size) 410 { 411 struct vgic_dist *d = &kvm->arch.vgic; 412 413 return (base + size > d->vgic_dist_base) && 414 (base < d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE); 415 } 416 417 bool vgic_lpis_enabled(struct kvm_vcpu *vcpu); 418 int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its, 419 u32 devid, u32 eventid, struct vgic_irq **irq); 420 struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi); 421 int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi); 422 void vgic_its_invalidate_all_caches(struct kvm *kvm); 423 424 /* GICv4.1 MMIO interface */ 425 int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq); 426 int vgic_its_invall(struct kvm_vcpu *vcpu); 427 428 bool system_supports_direct_sgis(void); 429 bool vgic_supports_direct_msis(struct kvm *kvm); 430 bool vgic_supports_direct_sgis(struct kvm *kvm); 431 432 static inline bool vgic_supports_direct_irqs(struct kvm *kvm) 433 { 434 return vgic_supports_direct_msis(kvm) || vgic_supports_direct_sgis(kvm); 435 } 436 437 int vgic_v4_init(struct kvm *kvm); 438 void vgic_v4_teardown(struct kvm *kvm); 439 void vgic_v4_configure_vsgis(struct kvm *kvm); 440 void vgic_v4_get_vlpi_state(struct vgic_irq *irq, bool *val); 441 int vgic_v4_request_vpe_irq(struct kvm_vcpu *vcpu, int irq); 442 443 void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu); 444 445 static inline bool kvm_has_gicv3(struct kvm *kvm) 446 { 447 return kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP); 448 } 449 450 void vgic_v3_flush_nested(struct kvm_vcpu *vcpu); 451 void vgic_v3_sync_nested(struct kvm_vcpu *vcpu); 452 void vgic_v3_load_nested(struct kvm_vcpu *vcpu); 453 void vgic_v3_put_nested(struct kvm_vcpu *vcpu); 454 void vgic_v3_handle_nested_maint_irq(struct kvm_vcpu *vcpu); 455 void vgic_v3_nested_update_mi(struct kvm_vcpu *vcpu); 456 457 static inline bool vgic_is_v3_compat(struct kvm *kvm) 458 { 459 return cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF) && 460 kvm_vgic_global_state.has_gcie_v3_compat; 461 } 462 463 static inline bool vgic_is_v3(struct kvm *kvm) 464 { 465 return kvm_vgic_global_state.type == VGIC_V3 || vgic_is_v3_compat(kvm); 466 } 467 468 int vgic_its_debug_init(struct kvm_device *dev); 469 void vgic_its_debug_destroy(struct kvm_device *dev); 470 471 #endif 472