1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright 2019 Arm Limited 4 * Author: Andrew Murray <Andrew.Murray@arm.com> 5 */ 6 #include <linux/kvm_host.h> 7 #include <linux/perf_event.h> 8 #include <linux/perf/arm_pmu.h> 9 #include <linux/perf/arm_pmuv3.h> 10 11 static DEFINE_PER_CPU(struct kvm_pmu_events, kvm_pmu_events); 12 13 /* 14 * Given the perf event attributes and system type, determine 15 * if we are going to need to switch counters at guest entry/exit. 16 */ 17 static bool kvm_pmu_switch_needed(struct perf_event_attr *attr) 18 { 19 /** 20 * With VHE the guest kernel runs at EL1 and the host at EL2, 21 * where user (EL0) is excluded then we have no reason to switch 22 * counters. 23 */ 24 if (has_vhe() && attr->exclude_user) 25 return false; 26 27 /* Only switch if attributes are different */ 28 return (attr->exclude_host != attr->exclude_guest); 29 } 30 31 struct kvm_pmu_events *kvm_get_pmu_events(void) 32 { 33 return this_cpu_ptr(&kvm_pmu_events); 34 } 35 36 /* 37 * Add events to track that we may want to switch at guest entry/exit 38 * time. 39 */ 40 void kvm_set_pmu_events(u64 set, struct perf_event_attr *attr) 41 { 42 struct kvm_pmu_events *pmu = kvm_get_pmu_events(); 43 44 if (!kvm_arm_support_pmu_v3() || !kvm_pmu_switch_needed(attr)) 45 return; 46 47 if (!attr->exclude_host) 48 pmu->events_host |= set; 49 if (!attr->exclude_guest) 50 pmu->events_guest |= set; 51 } 52 53 /* 54 * Stop tracking events 55 */ 56 void kvm_clr_pmu_events(u64 clr) 57 { 58 struct kvm_pmu_events *pmu = kvm_get_pmu_events(); 59 60 if (!kvm_arm_support_pmu_v3()) 61 return; 62 63 pmu->events_host &= ~clr; 64 pmu->events_guest &= ~clr; 65 } 66 67 /* 68 * Read a value direct from PMEVTYPER<idx> where idx is 0-30 69 * or PMxCFILTR_EL0 where idx is 31-32. 70 */ 71 static u64 kvm_vcpu_pmu_read_evtype_direct(int idx) 72 { 73 if (idx == ARMV8_PMU_CYCLE_IDX) 74 return read_pmccfiltr(); 75 else if (idx == ARMV8_PMU_INSTR_IDX) 76 return read_pmicfiltr(); 77 78 return read_pmevtypern(idx); 79 } 80 81 /* 82 * Write a value direct to PMEVTYPER<idx> where idx is 0-30 83 * or PMxCFILTR_EL0 where idx is 31-32. 84 */ 85 static void kvm_vcpu_pmu_write_evtype_direct(int idx, u32 val) 86 { 87 if (idx == ARMV8_PMU_CYCLE_IDX) 88 write_pmccfiltr(val); 89 else if (idx == ARMV8_PMU_INSTR_IDX) 90 write_pmicfiltr(val); 91 else 92 write_pmevtypern(idx, val); 93 } 94 95 /* 96 * Modify ARMv8 PMU events to include EL0 counting 97 */ 98 static void kvm_vcpu_pmu_enable_el0(unsigned long events) 99 { 100 u64 typer; 101 u32 counter; 102 103 for_each_set_bit(counter, &events, ARMPMU_MAX_HWEVENTS) { 104 typer = kvm_vcpu_pmu_read_evtype_direct(counter); 105 typer &= ~ARMV8_PMU_EXCLUDE_EL0; 106 kvm_vcpu_pmu_write_evtype_direct(counter, typer); 107 } 108 } 109 110 /* 111 * Modify ARMv8 PMU events to exclude EL0 counting 112 */ 113 static void kvm_vcpu_pmu_disable_el0(unsigned long events) 114 { 115 u64 typer; 116 u32 counter; 117 118 for_each_set_bit(counter, &events, ARMPMU_MAX_HWEVENTS) { 119 typer = kvm_vcpu_pmu_read_evtype_direct(counter); 120 typer |= ARMV8_PMU_EXCLUDE_EL0; 121 kvm_vcpu_pmu_write_evtype_direct(counter, typer); 122 } 123 } 124 125 /* 126 * On VHE ensure that only guest events have EL0 counting enabled. 127 * This is called from both vcpu_{load,put} and the sysreg handling. 128 * Since the latter is preemptible, special care must be taken to 129 * disable preemption. 130 */ 131 void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) 132 { 133 struct kvm_pmu_events *pmu; 134 u64 events_guest, events_host; 135 136 if (!kvm_arm_support_pmu_v3() || !has_vhe()) 137 return; 138 139 preempt_disable(); 140 pmu = kvm_get_pmu_events(); 141 events_guest = pmu->events_guest; 142 events_host = pmu->events_host; 143 144 kvm_vcpu_pmu_enable_el0(events_guest); 145 kvm_vcpu_pmu_disable_el0(events_host); 146 preempt_enable(); 147 } 148 149 /* 150 * On VHE ensure that only host events have EL0 counting enabled 151 */ 152 void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) 153 { 154 struct kvm_pmu_events *pmu; 155 u64 events_guest, events_host; 156 157 if (!kvm_arm_support_pmu_v3() || !has_vhe()) 158 return; 159 160 pmu = kvm_get_pmu_events(); 161 events_guest = pmu->events_guest; 162 events_host = pmu->events_host; 163 164 kvm_vcpu_pmu_enable_el0(events_host); 165 kvm_vcpu_pmu_disable_el0(events_guest); 166 } 167 168 /* 169 * With VHE, keep track of the PMUSERENR_EL0 value for the host EL0 on the pCPU 170 * where PMUSERENR_EL0 for the guest is loaded, since PMUSERENR_EL0 is switched 171 * to the value for the guest on vcpu_load(). The value for the host EL0 172 * will be restored on vcpu_put(), before returning to userspace. 173 * This isn't necessary for nVHE, as the register is context switched for 174 * every guest enter/exit. 175 * 176 * Return true if KVM takes care of the register. Otherwise return false. 177 */ 178 bool kvm_set_pmuserenr(u64 val) 179 { 180 struct kvm_cpu_context *hctxt; 181 struct kvm_vcpu *vcpu; 182 183 if (!kvm_arm_support_pmu_v3() || !has_vhe()) 184 return false; 185 186 vcpu = kvm_get_running_vcpu(); 187 if (!vcpu || !vcpu_get_flag(vcpu, PMUSERENR_ON_CPU)) 188 return false; 189 190 hctxt = host_data_ptr(host_ctxt); 191 ctxt_sys_reg(hctxt, PMUSERENR_EL0) = val; 192 return true; 193 } 194 195 /* 196 * If we interrupted the guest to update the host PMU context, make 197 * sure we re-apply the guest EL0 state. 198 */ 199 void kvm_vcpu_pmu_resync_el0(void) 200 { 201 struct kvm_vcpu *vcpu; 202 203 if (!has_vhe() || !in_interrupt()) 204 return; 205 206 vcpu = kvm_get_running_vcpu(); 207 if (!vcpu) 208 return; 209 210 kvm_make_request(KVM_REQ_RESYNC_PMU_EL0, vcpu); 211 } 212