1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 - Google LLC 4 * Author: David Brazdil <dbrazdil@google.com> 5 */ 6 7 #include <asm/kvm_asm.h> 8 #include <asm/kvm_hyp.h> 9 #include <asm/kvm_mmu.h> 10 #include <linux/arm-smccc.h> 11 #include <linux/kvm_host.h> 12 #include <uapi/linux/psci.h> 13 14 #include <nvhe/trap_handler.h> 15 16 void kvm_hyp_cpu_entry(unsigned long r0); 17 void kvm_hyp_cpu_resume(unsigned long r0); 18 19 void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); 20 21 /* Config options set by the host. */ 22 struct kvm_host_psci_config __ro_after_init kvm_host_psci_config; 23 s64 __ro_after_init hyp_physvirt_offset; 24 25 #define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset) 26 27 #define INVALID_CPU_ID UINT_MAX 28 29 struct psci_boot_args { 30 atomic_t lock; 31 unsigned long pc; 32 unsigned long r0; 33 }; 34 35 #define PSCI_BOOT_ARGS_UNLOCKED 0 36 #define PSCI_BOOT_ARGS_LOCKED 1 37 38 #define PSCI_BOOT_ARGS_INIT \ 39 ((struct psci_boot_args){ \ 40 .lock = ATOMIC_INIT(PSCI_BOOT_ARGS_UNLOCKED), \ 41 }) 42 43 static DEFINE_PER_CPU(struct psci_boot_args, cpu_on_args) = PSCI_BOOT_ARGS_INIT; 44 static DEFINE_PER_CPU(struct psci_boot_args, suspend_args) = PSCI_BOOT_ARGS_INIT; 45 46 #define is_psci_0_1(what, func_id) \ 47 (kvm_host_psci_config.psci_0_1_ ## what ## _implemented && \ 48 (func_id) == kvm_host_psci_config.function_ids_0_1.what) 49 50 static bool is_psci_0_1_call(u64 func_id) 51 { 52 return (is_psci_0_1(cpu_suspend, func_id) || 53 is_psci_0_1(cpu_on, func_id) || 54 is_psci_0_1(cpu_off, func_id) || 55 is_psci_0_1(migrate, func_id)); 56 } 57 58 static bool is_psci_0_2_call(u64 func_id) 59 { 60 /* SMCCC reserves IDs 0x00-1F with the given 32/64-bit base for PSCI. */ 61 return (PSCI_0_2_FN(0) <= func_id && func_id <= PSCI_0_2_FN(31)) || 62 (PSCI_0_2_FN64(0) <= func_id && func_id <= PSCI_0_2_FN64(31)); 63 } 64 65 static unsigned long psci_call(unsigned long fn, unsigned long arg0, 66 unsigned long arg1, unsigned long arg2) 67 { 68 struct arm_smccc_res res; 69 70 arm_smccc_1_1_smc(fn, arg0, arg1, arg2, &res); 71 return res.a0; 72 } 73 74 static unsigned long psci_forward(struct kvm_cpu_context *host_ctxt) 75 { 76 return psci_call(cpu_reg(host_ctxt, 0), cpu_reg(host_ctxt, 1), 77 cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3)); 78 } 79 80 static __noreturn unsigned long psci_forward_noreturn(struct kvm_cpu_context *host_ctxt) 81 { 82 psci_forward(host_ctxt); 83 hyp_panic(); /* unreachable */ 84 } 85 86 static unsigned int find_cpu_id(u64 mpidr) 87 { 88 unsigned int i; 89 90 /* Reject invalid MPIDRs */ 91 if (mpidr & ~MPIDR_HWID_BITMASK) 92 return INVALID_CPU_ID; 93 94 for (i = 0; i < NR_CPUS; i++) { 95 if (cpu_logical_map(i) == mpidr) 96 return i; 97 } 98 99 return INVALID_CPU_ID; 100 } 101 102 static __always_inline bool try_acquire_boot_args(struct psci_boot_args *args) 103 { 104 return atomic_cmpxchg_acquire(&args->lock, 105 PSCI_BOOT_ARGS_UNLOCKED, 106 PSCI_BOOT_ARGS_LOCKED) == 107 PSCI_BOOT_ARGS_UNLOCKED; 108 } 109 110 static __always_inline void release_boot_args(struct psci_boot_args *args) 111 { 112 atomic_set_release(&args->lock, PSCI_BOOT_ARGS_UNLOCKED); 113 } 114 115 static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt) 116 { 117 DECLARE_REG(u64, mpidr, host_ctxt, 1); 118 DECLARE_REG(unsigned long, pc, host_ctxt, 2); 119 DECLARE_REG(unsigned long, r0, host_ctxt, 3); 120 121 unsigned int cpu_id; 122 struct psci_boot_args *boot_args; 123 struct kvm_nvhe_init_params *init_params; 124 int ret; 125 126 /* 127 * Find the logical CPU ID for the given MPIDR. The search set is 128 * the set of CPUs that were online at the point of KVM initialization. 129 * Booting other CPUs is rejected because their cpufeatures were not 130 * checked against the finalized capabilities. This could be relaxed 131 * by doing the feature checks in hyp. 132 */ 133 cpu_id = find_cpu_id(mpidr); 134 if (cpu_id == INVALID_CPU_ID) 135 return PSCI_RET_INVALID_PARAMS; 136 137 boot_args = per_cpu_ptr(hyp_symbol_addr(cpu_on_args), cpu_id); 138 init_params = per_cpu_ptr(hyp_symbol_addr(kvm_init_params), cpu_id); 139 140 /* Check if the target CPU is already being booted. */ 141 if (!try_acquire_boot_args(boot_args)) 142 return PSCI_RET_ALREADY_ON; 143 144 boot_args->pc = pc; 145 boot_args->r0 = r0; 146 wmb(); 147 148 ret = psci_call(func_id, mpidr, 149 __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_entry)), 150 __hyp_pa(init_params)); 151 152 /* If successful, the lock will be released by the target CPU. */ 153 if (ret != PSCI_RET_SUCCESS) 154 release_boot_args(boot_args); 155 156 return ret; 157 } 158 159 static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) 160 { 161 DECLARE_REG(u64, power_state, host_ctxt, 1); 162 DECLARE_REG(unsigned long, pc, host_ctxt, 2); 163 DECLARE_REG(unsigned long, r0, host_ctxt, 3); 164 165 struct psci_boot_args *boot_args; 166 struct kvm_nvhe_init_params *init_params; 167 168 boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args)); 169 init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params)); 170 171 /* 172 * No need to acquire a lock before writing to boot_args because a core 173 * can only suspend itself. Racy CPU_ON calls use a separate struct. 174 */ 175 boot_args->pc = pc; 176 boot_args->r0 = r0; 177 178 /* 179 * Will either return if shallow sleep state, or wake up into the entry 180 * point if it is a deep sleep state. 181 */ 182 return psci_call(func_id, power_state, 183 __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)), 184 __hyp_pa(init_params)); 185 } 186 187 static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt) 188 { 189 DECLARE_REG(unsigned long, pc, host_ctxt, 1); 190 DECLARE_REG(unsigned long, r0, host_ctxt, 2); 191 192 struct psci_boot_args *boot_args; 193 struct kvm_nvhe_init_params *init_params; 194 195 boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args)); 196 init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params)); 197 198 /* 199 * No need to acquire a lock before writing to boot_args because a core 200 * can only suspend itself. Racy CPU_ON calls use a separate struct. 201 */ 202 boot_args->pc = pc; 203 boot_args->r0 = r0; 204 205 /* Will only return on error. */ 206 return psci_call(func_id, 207 __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)), 208 __hyp_pa(init_params), 0); 209 } 210 211 asmlinkage void __noreturn kvm_host_psci_cpu_entry(bool is_cpu_on) 212 { 213 struct psci_boot_args *boot_args; 214 struct kvm_cpu_context *host_ctxt; 215 216 host_ctxt = &this_cpu_ptr(hyp_symbol_addr(kvm_host_data))->host_ctxt; 217 218 if (is_cpu_on) 219 boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_on_args)); 220 else 221 boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args)); 222 223 cpu_reg(host_ctxt, 0) = boot_args->r0; 224 write_sysreg_el2(boot_args->pc, SYS_ELR); 225 226 if (is_cpu_on) 227 release_boot_args(boot_args); 228 229 __host_enter(host_ctxt); 230 } 231 232 static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 233 { 234 if (is_psci_0_1(cpu_off, func_id) || is_psci_0_1(migrate, func_id)) 235 return psci_forward(host_ctxt); 236 if (is_psci_0_1(cpu_on, func_id)) 237 return psci_cpu_on(func_id, host_ctxt); 238 if (is_psci_0_1(cpu_suspend, func_id)) 239 return psci_cpu_suspend(func_id, host_ctxt); 240 241 return PSCI_RET_NOT_SUPPORTED; 242 } 243 244 static unsigned long psci_0_2_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 245 { 246 switch (func_id) { 247 case PSCI_0_2_FN_PSCI_VERSION: 248 case PSCI_0_2_FN_CPU_OFF: 249 case PSCI_0_2_FN64_AFFINITY_INFO: 250 case PSCI_0_2_FN64_MIGRATE: 251 case PSCI_0_2_FN_MIGRATE_INFO_TYPE: 252 case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU: 253 return psci_forward(host_ctxt); 254 case PSCI_0_2_FN_SYSTEM_OFF: 255 case PSCI_0_2_FN_SYSTEM_RESET: 256 psci_forward_noreturn(host_ctxt); 257 unreachable(); 258 case PSCI_0_2_FN64_CPU_SUSPEND: 259 return psci_cpu_suspend(func_id, host_ctxt); 260 case PSCI_0_2_FN64_CPU_ON: 261 return psci_cpu_on(func_id, host_ctxt); 262 default: 263 return PSCI_RET_NOT_SUPPORTED; 264 } 265 } 266 267 static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context *host_ctxt) 268 { 269 switch (func_id) { 270 case PSCI_1_0_FN_PSCI_FEATURES: 271 case PSCI_1_0_FN_SET_SUSPEND_MODE: 272 case PSCI_1_1_FN64_SYSTEM_RESET2: 273 return psci_forward(host_ctxt); 274 case PSCI_1_0_FN64_SYSTEM_SUSPEND: 275 return psci_system_suspend(func_id, host_ctxt); 276 default: 277 return psci_0_2_handler(func_id, host_ctxt); 278 } 279 } 280 281 bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt) 282 { 283 DECLARE_REG(u64, func_id, host_ctxt, 0); 284 unsigned long ret; 285 286 switch (kvm_host_psci_config.version) { 287 case PSCI_VERSION(0, 1): 288 if (!is_psci_0_1_call(func_id)) 289 return false; 290 ret = psci_0_1_handler(func_id, host_ctxt); 291 break; 292 case PSCI_VERSION(0, 2): 293 if (!is_psci_0_2_call(func_id)) 294 return false; 295 ret = psci_0_2_handler(func_id, host_ctxt); 296 break; 297 default: 298 if (!is_psci_0_2_call(func_id)) 299 return false; 300 ret = psci_1_0_handler(func_id, host_ctxt); 301 break; 302 } 303 304 cpu_reg(host_ctxt, 0) = ret; 305 cpu_reg(host_ctxt, 1) = 0; 306 cpu_reg(host_ctxt, 2) = 0; 307 cpu_reg(host_ctxt, 3) = 0; 308 return true; 309 } 310