1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 - ARM Ltd 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #include <linux/arm-smccc.h> 8 #include <linux/preempt.h> 9 #include <linux/kvm_host.h> 10 #include <linux/uaccess.h> 11 #include <linux/wait.h> 12 13 #include <asm/cputype.h> 14 #include <asm/kvm_emulate.h> 15 16 #include <kvm/arm_psci.h> 17 #include <kvm/arm_hypercalls.h> 18 19 /* 20 * This is an implementation of the Power State Coordination Interface 21 * as described in ARM document number ARM DEN 0022A. 22 */ 23 24 #define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1) 25 26 static unsigned long psci_affinity_mask(unsigned long affinity_level) 27 { 28 if (affinity_level <= 3) 29 return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level); 30 31 return 0; 32 } 33 34 static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) 35 { 36 /* 37 * NOTE: For simplicity, we make VCPU suspend emulation to be 38 * same-as WFI (Wait-for-interrupt) emulation. 39 * 40 * This means for KVM the wakeup events are interrupts and 41 * this is consistent with intended use of StateID as described 42 * in section 5.4.1 of PSCI v0.2 specification (ARM DEN 0022A). 43 * 44 * Further, we also treat power-down request to be same as 45 * stand-by request as-per section 5.4.2 clause 3 of PSCI v0.2 46 * specification (ARM DEN 0022A). This means all suspend states 47 * for KVM will preserve the register state. 48 */ 49 kvm_vcpu_wfi(vcpu); 50 51 return PSCI_RET_SUCCESS; 52 } 53 54 static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, 55 unsigned long affinity) 56 { 57 return !(affinity & ~MPIDR_HWID_BITMASK); 58 } 59 60 static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) 61 { 62 struct vcpu_reset_state *reset_state; 63 struct kvm *kvm = source_vcpu->kvm; 64 struct kvm_vcpu *vcpu = NULL; 65 int ret = PSCI_RET_SUCCESS; 66 unsigned long cpu_id; 67 68 cpu_id = smccc_get_arg1(source_vcpu); 69 if (!kvm_psci_valid_affinity(source_vcpu, cpu_id)) 70 return PSCI_RET_INVALID_PARAMS; 71 72 vcpu = kvm_mpidr_to_vcpu(kvm, cpu_id); 73 74 /* 75 * Make sure the caller requested a valid CPU and that the CPU is 76 * turned off. 77 */ 78 if (!vcpu) 79 return PSCI_RET_INVALID_PARAMS; 80 81 spin_lock(&vcpu->arch.mp_state_lock); 82 if (!kvm_arm_vcpu_stopped(vcpu)) { 83 if (kvm_psci_version(source_vcpu) != KVM_ARM_PSCI_0_1) 84 ret = PSCI_RET_ALREADY_ON; 85 else 86 ret = PSCI_RET_INVALID_PARAMS; 87 88 goto out_unlock; 89 } 90 91 reset_state = &vcpu->arch.reset_state; 92 93 reset_state->pc = smccc_get_arg2(source_vcpu); 94 95 /* Propagate caller endianness */ 96 reset_state->be = kvm_vcpu_is_be(source_vcpu); 97 98 /* 99 * NOTE: We always update r0 (or x0) because for PSCI v0.1 100 * the general purpose registers are undefined upon CPU_ON. 101 */ 102 reset_state->r0 = smccc_get_arg3(source_vcpu); 103 104 reset_state->reset = true; 105 kvm_make_request(KVM_REQ_VCPU_RESET, vcpu); 106 107 /* 108 * Make sure the reset request is observed if the RUNNABLE mp_state is 109 * observed. 110 */ 111 smp_wmb(); 112 113 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); 114 kvm_vcpu_wake_up(vcpu); 115 116 out_unlock: 117 spin_unlock(&vcpu->arch.mp_state_lock); 118 return ret; 119 } 120 121 static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) 122 { 123 int matching_cpus = 0; 124 unsigned long i, mpidr; 125 unsigned long target_affinity; 126 unsigned long target_affinity_mask; 127 unsigned long lowest_affinity_level; 128 struct kvm *kvm = vcpu->kvm; 129 struct kvm_vcpu *tmp; 130 131 target_affinity = smccc_get_arg1(vcpu); 132 lowest_affinity_level = smccc_get_arg2(vcpu); 133 134 if (!kvm_psci_valid_affinity(vcpu, target_affinity)) 135 return PSCI_RET_INVALID_PARAMS; 136 137 /* Determine target affinity mask */ 138 target_affinity_mask = psci_affinity_mask(lowest_affinity_level); 139 if (!target_affinity_mask) 140 return PSCI_RET_INVALID_PARAMS; 141 142 /* Ignore other bits of target affinity */ 143 target_affinity &= target_affinity_mask; 144 145 /* 146 * If one or more VCPU matching target affinity are running 147 * then ON else OFF 148 */ 149 kvm_for_each_vcpu(i, tmp, kvm) { 150 mpidr = kvm_vcpu_get_mpidr_aff(tmp); 151 if ((mpidr & target_affinity_mask) == target_affinity) { 152 matching_cpus++; 153 if (!kvm_arm_vcpu_stopped(tmp)) 154 return PSCI_0_2_AFFINITY_LEVEL_ON; 155 } 156 } 157 158 if (!matching_cpus) 159 return PSCI_RET_INVALID_PARAMS; 160 161 return PSCI_0_2_AFFINITY_LEVEL_OFF; 162 } 163 164 static void kvm_prepare_system_event(struct kvm_vcpu *vcpu, u32 type, u64 flags) 165 { 166 unsigned long i; 167 struct kvm_vcpu *tmp; 168 169 /* 170 * The KVM ABI specifies that a system event exit may call KVM_RUN 171 * again and may perform shutdown/reboot at a later time that when the 172 * actual request is made. Since we are implementing PSCI and a 173 * caller of PSCI reboot and shutdown expects that the system shuts 174 * down or reboots immediately, let's make sure that VCPUs are not run 175 * after this call is handled and before the VCPUs have been 176 * re-initialized. 177 */ 178 kvm_for_each_vcpu(i, tmp, vcpu->kvm) { 179 spin_lock(&tmp->arch.mp_state_lock); 180 WRITE_ONCE(tmp->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); 181 spin_unlock(&tmp->arch.mp_state_lock); 182 } 183 kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_SLEEP); 184 185 memset(&vcpu->run->system_event, 0, sizeof(vcpu->run->system_event)); 186 vcpu->run->system_event.type = type; 187 vcpu->run->system_event.ndata = 1; 188 vcpu->run->system_event.data[0] = flags; 189 vcpu->run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 190 } 191 192 static void kvm_psci_system_off(struct kvm_vcpu *vcpu) 193 { 194 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_SHUTDOWN, 0); 195 } 196 197 static void kvm_psci_system_reset(struct kvm_vcpu *vcpu) 198 { 199 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 0); 200 } 201 202 static void kvm_psci_system_reset2(struct kvm_vcpu *vcpu) 203 { 204 kvm_prepare_system_event(vcpu, KVM_SYSTEM_EVENT_RESET, 205 KVM_SYSTEM_EVENT_RESET_FLAG_PSCI_RESET2); 206 } 207 208 static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu) 209 { 210 struct kvm_run *run = vcpu->run; 211 212 memset(&run->system_event, 0, sizeof(vcpu->run->system_event)); 213 run->system_event.type = KVM_SYSTEM_EVENT_SUSPEND; 214 run->exit_reason = KVM_EXIT_SYSTEM_EVENT; 215 } 216 217 static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) 218 { 219 int i; 220 221 /* 222 * Zero the input registers' upper 32 bits. They will be fully 223 * zeroed on exit, so we're fine changing them in place. 224 */ 225 for (i = 1; i < 4; i++) 226 vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); 227 } 228 229 static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn) 230 { 231 /* 232 * Prevent 32 bit guests from calling 64 bit PSCI functions. 233 */ 234 if ((fn & PSCI_0_2_64BIT) && vcpu_mode_is_32bit(vcpu)) 235 return PSCI_RET_NOT_SUPPORTED; 236 237 return 0; 238 } 239 240 static int kvm_psci_0_2_call(struct kvm_vcpu *vcpu) 241 { 242 u32 psci_fn = smccc_get_function(vcpu); 243 unsigned long val; 244 int ret = 1; 245 246 switch (psci_fn) { 247 case PSCI_0_2_FN_PSCI_VERSION: 248 /* 249 * Bits[31:16] = Major Version = 0 250 * Bits[15:0] = Minor Version = 2 251 */ 252 val = KVM_ARM_PSCI_0_2; 253 break; 254 case PSCI_0_2_FN_CPU_SUSPEND: 255 case PSCI_0_2_FN64_CPU_SUSPEND: 256 val = kvm_psci_vcpu_suspend(vcpu); 257 break; 258 case PSCI_0_2_FN_CPU_OFF: 259 kvm_arm_vcpu_power_off(vcpu); 260 val = PSCI_RET_SUCCESS; 261 break; 262 case PSCI_0_2_FN_CPU_ON: 263 kvm_psci_narrow_to_32bit(vcpu); 264 fallthrough; 265 case PSCI_0_2_FN64_CPU_ON: 266 val = kvm_psci_vcpu_on(vcpu); 267 break; 268 case PSCI_0_2_FN_AFFINITY_INFO: 269 kvm_psci_narrow_to_32bit(vcpu); 270 fallthrough; 271 case PSCI_0_2_FN64_AFFINITY_INFO: 272 val = kvm_psci_vcpu_affinity_info(vcpu); 273 break; 274 case PSCI_0_2_FN_MIGRATE_INFO_TYPE: 275 /* 276 * Trusted OS is MP hence does not require migration 277 * or 278 * Trusted OS is not present 279 */ 280 val = PSCI_0_2_TOS_MP; 281 break; 282 case PSCI_0_2_FN_SYSTEM_OFF: 283 kvm_psci_system_off(vcpu); 284 /* 285 * We shouldn't be going back to guest VCPU after 286 * receiving SYSTEM_OFF request. 287 * 288 * If user space accidentally/deliberately resumes 289 * guest VCPU after SYSTEM_OFF request then guest 290 * VCPU should see internal failure from PSCI return 291 * value. To achieve this, we preload r0 (or x0) with 292 * PSCI return value INTERNAL_FAILURE. 293 */ 294 val = PSCI_RET_INTERNAL_FAILURE; 295 ret = 0; 296 break; 297 case PSCI_0_2_FN_SYSTEM_RESET: 298 kvm_psci_system_reset(vcpu); 299 /* 300 * Same reason as SYSTEM_OFF for preloading r0 (or x0) 301 * with PSCI return value INTERNAL_FAILURE. 302 */ 303 val = PSCI_RET_INTERNAL_FAILURE; 304 ret = 0; 305 break; 306 default: 307 val = PSCI_RET_NOT_SUPPORTED; 308 break; 309 } 310 311 smccc_set_retval(vcpu, val, 0, 0, 0); 312 return ret; 313 } 314 315 static int kvm_psci_1_x_call(struct kvm_vcpu *vcpu, u32 minor) 316 { 317 unsigned long val = PSCI_RET_NOT_SUPPORTED; 318 u32 psci_fn = smccc_get_function(vcpu); 319 struct kvm *kvm = vcpu->kvm; 320 u32 arg; 321 int ret = 1; 322 323 switch(psci_fn) { 324 case PSCI_0_2_FN_PSCI_VERSION: 325 val = minor == 0 ? KVM_ARM_PSCI_1_0 : KVM_ARM_PSCI_1_1; 326 break; 327 case PSCI_1_0_FN_PSCI_FEATURES: 328 arg = smccc_get_arg1(vcpu); 329 val = kvm_psci_check_allowed_function(vcpu, arg); 330 if (val) 331 break; 332 333 val = PSCI_RET_NOT_SUPPORTED; 334 335 switch(arg) { 336 case PSCI_0_2_FN_PSCI_VERSION: 337 case PSCI_0_2_FN_CPU_SUSPEND: 338 case PSCI_0_2_FN64_CPU_SUSPEND: 339 case PSCI_0_2_FN_CPU_OFF: 340 case PSCI_0_2_FN_CPU_ON: 341 case PSCI_0_2_FN64_CPU_ON: 342 case PSCI_0_2_FN_AFFINITY_INFO: 343 case PSCI_0_2_FN64_AFFINITY_INFO: 344 case PSCI_0_2_FN_MIGRATE_INFO_TYPE: 345 case PSCI_0_2_FN_SYSTEM_OFF: 346 case PSCI_0_2_FN_SYSTEM_RESET: 347 case PSCI_1_0_FN_PSCI_FEATURES: 348 case ARM_SMCCC_VERSION_FUNC_ID: 349 val = 0; 350 break; 351 case PSCI_1_0_FN_SYSTEM_SUSPEND: 352 case PSCI_1_0_FN64_SYSTEM_SUSPEND: 353 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) 354 val = 0; 355 break; 356 case PSCI_1_1_FN_SYSTEM_RESET2: 357 case PSCI_1_1_FN64_SYSTEM_RESET2: 358 if (minor >= 1) 359 val = 0; 360 break; 361 } 362 break; 363 case PSCI_1_0_FN_SYSTEM_SUSPEND: 364 kvm_psci_narrow_to_32bit(vcpu); 365 fallthrough; 366 case PSCI_1_0_FN64_SYSTEM_SUSPEND: 367 /* 368 * Return directly to userspace without changing the vCPU's 369 * registers. Userspace depends on reading the SMCCC parameters 370 * to implement SYSTEM_SUSPEND. 371 */ 372 if (test_bit(KVM_ARCH_FLAG_SYSTEM_SUSPEND_ENABLED, &kvm->arch.flags)) { 373 kvm_psci_system_suspend(vcpu); 374 return 0; 375 } 376 break; 377 case PSCI_1_1_FN_SYSTEM_RESET2: 378 kvm_psci_narrow_to_32bit(vcpu); 379 fallthrough; 380 case PSCI_1_1_FN64_SYSTEM_RESET2: 381 if (minor >= 1) { 382 arg = smccc_get_arg1(vcpu); 383 384 if (arg <= PSCI_1_1_RESET_TYPE_SYSTEM_WARM_RESET || 385 arg >= PSCI_1_1_RESET_TYPE_VENDOR_START) { 386 kvm_psci_system_reset2(vcpu); 387 vcpu_set_reg(vcpu, 0, PSCI_RET_INTERNAL_FAILURE); 388 return 0; 389 } 390 391 val = PSCI_RET_INVALID_PARAMS; 392 break; 393 } 394 break; 395 default: 396 return kvm_psci_0_2_call(vcpu); 397 } 398 399 smccc_set_retval(vcpu, val, 0, 0, 0); 400 return ret; 401 } 402 403 static int kvm_psci_0_1_call(struct kvm_vcpu *vcpu) 404 { 405 u32 psci_fn = smccc_get_function(vcpu); 406 unsigned long val; 407 408 switch (psci_fn) { 409 case KVM_PSCI_FN_CPU_OFF: 410 kvm_arm_vcpu_power_off(vcpu); 411 val = PSCI_RET_SUCCESS; 412 break; 413 case KVM_PSCI_FN_CPU_ON: 414 val = kvm_psci_vcpu_on(vcpu); 415 break; 416 default: 417 val = PSCI_RET_NOT_SUPPORTED; 418 break; 419 } 420 421 smccc_set_retval(vcpu, val, 0, 0, 0); 422 return 1; 423 } 424 425 /** 426 * kvm_psci_call - handle PSCI call if r0 value is in range 427 * @vcpu: Pointer to the VCPU struct 428 * 429 * Handle PSCI calls from guests through traps from HVC instructions. 430 * The calling convention is similar to SMC calls to the secure world 431 * where the function number is placed in r0. 432 * 433 * This function returns: > 0 (success), 0 (success but exit to user 434 * space), and < 0 (errors) 435 * 436 * Errors: 437 * -EINVAL: Unrecognized PSCI function 438 */ 439 int kvm_psci_call(struct kvm_vcpu *vcpu) 440 { 441 u32 psci_fn = smccc_get_function(vcpu); 442 int version = kvm_psci_version(vcpu); 443 unsigned long val; 444 445 val = kvm_psci_check_allowed_function(vcpu, psci_fn); 446 if (val) { 447 smccc_set_retval(vcpu, val, 0, 0, 0); 448 return 1; 449 } 450 451 switch (version) { 452 case KVM_ARM_PSCI_1_1: 453 return kvm_psci_1_x_call(vcpu, 1); 454 case KVM_ARM_PSCI_1_0: 455 return kvm_psci_1_x_call(vcpu, 0); 456 case KVM_ARM_PSCI_0_2: 457 return kvm_psci_0_2_call(vcpu); 458 case KVM_ARM_PSCI_0_1: 459 return kvm_psci_0_1_call(vcpu); 460 default: 461 WARN_ONCE(1, "Unknown PSCI version %d", version); 462 smccc_set_retval(vcpu, SMCCC_RET_NOT_SUPPORTED, 0, 0, 0); 463 return 1; 464 } 465 } 466