1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/errno.h> 10 #include <linux/err.h> 11 #include <linux/module.h> 12 #include <linux/kvm_host.h> 13 #include <linux/cpu_pm.h> 14 #include <asm/cpufeature.h> 15 #include <asm/kvm_mmu.h> 16 #include <asm/kvm_nacl.h> 17 #include <asm/sbi.h> 18 #include <asm/kvm_vcpu_vector.h> 19 20 static DEFINE_PER_CPU(bool, kvm_riscv_virtualization_enabled); 21 22 DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa); 23 24 static void kvm_riscv_setup_vendor_features(void) 25 { 26 /* Andes AX66: split two-stage TLBs */ 27 if (riscv_cached_mvendorid(0) == ANDES_VENDOR_ID && 28 (riscv_cached_marchid(0) & 0xFFFF) == 0x8A66) { 29 static_branch_enable(&kvm_riscv_vsstage_tlb_no_gpa); 30 kvm_info("VS-stage TLB does not cache guest physical address and VMID\n"); 31 } 32 } 33 34 long kvm_arch_dev_ioctl(struct file *filp, 35 unsigned int ioctl, unsigned long arg) 36 { 37 return -EINVAL; 38 } 39 40 /* Initialize hypervisor CSRs - called during CPU online and non-retention idle resume */ 41 static void kvm_riscv_csr_init(void) 42 { 43 csr_write(CSR_HEDELEG, 0); 44 csr_write(CSR_HIDELEG, 0); 45 46 /* VS should access only the time counter directly. Everything else should trap */ 47 csr_write(CSR_HCOUNTEREN, 0x02); 48 49 csr_write(CSR_HVIP, 0); 50 } 51 52 /* Clear hypervisor CSRs - called during CPU offline and non-retention idle entry */ 53 static void kvm_riscv_csr_cleanup(void) 54 { 55 /* 56 * After clearing the hideleg CSR, the host kernel will receive 57 * spurious interrupts if hvip CSR has pending interrupts and the 58 * corresponding enable bits in vsie CSR are asserted. To avoid it, 59 * hvip CSR and vsie CSR must be cleared before clearing hideleg CSR. 60 */ 61 csr_write(CSR_VSIE, 0); 62 csr_write(CSR_HVIP, 0); 63 csr_write(CSR_HEDELEG, 0); 64 csr_write(CSR_HIDELEG, 0); 65 66 kvm_riscv_clear_former_vcpu(); 67 } 68 69 int kvm_arch_enable_virtualization_cpu(void) 70 { 71 int rc; 72 73 rc = kvm_riscv_nacl_enable(); 74 if (rc) 75 return rc; 76 77 kvm_riscv_csr_init(); 78 kvm_riscv_aia_enable(); 79 80 __this_cpu_write(kvm_riscv_virtualization_enabled, true); 81 82 return 0; 83 } 84 85 void kvm_arch_disable_virtualization_cpu(void) 86 { 87 kvm_riscv_aia_disable(); 88 kvm_riscv_csr_cleanup(); 89 kvm_riscv_nacl_disable(); 90 91 __this_cpu_write(kvm_riscv_virtualization_enabled, false); 92 } 93 94 static int kvm_riscv_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) 95 { 96 switch (cmd) { 97 case CPU_PM_EXIT: 98 case CPU_PM_ENTER_FAILED: 99 /* 100 * Only restore hypervisor state if KVM virtualization is 101 * enabled on this CPU. This prevents unintentional re-enabling 102 * of virtualization after it has been explicitly disabled. 103 */ 104 if (__this_cpu_read(kvm_riscv_virtualization_enabled)) { 105 kvm_riscv_csr_init(); 106 kvm_riscv_aia_pm_exit(); 107 } 108 return NOTIFY_OK; 109 case CPU_PM_ENTER: 110 /* 111 * Only save and clear hypervisor state if KVM virtualization 112 * is enabled on this CPU. 113 */ 114 if (__this_cpu_read(kvm_riscv_virtualization_enabled)) { 115 kvm_riscv_aia_pm_enter(); 116 kvm_riscv_csr_cleanup(); 117 } 118 return NOTIFY_OK; 119 default: 120 break; 121 } 122 123 return NOTIFY_DONE; 124 } 125 126 static struct notifier_block kvm_riscv_cpu_pm_nb = { 127 .notifier_call = kvm_riscv_cpu_pm_notifier, 128 }; 129 130 static void kvm_riscv_teardown(void) 131 { 132 kvm_riscv_aia_exit(); 133 kvm_riscv_nacl_exit(); 134 kvm_riscv_v_exit(); 135 kvm_unregister_perf_callbacks(); 136 } 137 138 static int __init riscv_kvm_init(void) 139 { 140 int rc; 141 char slist[64]; 142 const char *str; 143 144 if (!riscv_isa_extension_available(NULL, H)) { 145 kvm_info("hypervisor extension not available\n"); 146 return -ENODEV; 147 } 148 149 if (sbi_spec_is_0_1()) { 150 kvm_info("require SBI v0.2 or higher\n"); 151 return -ENODEV; 152 } 153 154 if (!sbi_probe_extension(SBI_EXT_RFENCE)) { 155 kvm_info("require SBI RFENCE extension\n"); 156 return -ENODEV; 157 } 158 159 rc = kvm_riscv_nacl_init(); 160 if (rc && rc != -ENODEV) 161 return rc; 162 163 kvm_riscv_gstage_mode_detect(); 164 switch (kvm_riscv_gstage_max_pgd_levels) { 165 case 2: 166 str = "Sv32x4"; 167 break; 168 case 3: 169 str = "Sv39x4"; 170 break; 171 case 4: 172 str = "Sv48x4"; 173 break; 174 case 5: 175 str = "Sv57x4"; 176 break; 177 default: 178 kvm_riscv_nacl_exit(); 179 return -ENODEV; 180 } 181 182 kvm_riscv_gstage_vmid_detect(); 183 184 rc = kvm_riscv_aia_init(); 185 if (rc && rc != -ENODEV) { 186 kvm_riscv_nacl_exit(); 187 return rc; 188 } 189 190 kvm_info("hypervisor extension available\n"); 191 192 if (kvm_riscv_nacl_available()) { 193 rc = 0; 194 slist[0] = '\0'; 195 if (kvm_riscv_nacl_sync_csr_available()) { 196 if (rc) 197 strcat(slist, ", "); 198 strcat(slist, "sync_csr"); 199 rc++; 200 } 201 if (kvm_riscv_nacl_sync_hfence_available()) { 202 if (rc) 203 strcat(slist, ", "); 204 strcat(slist, "sync_hfence"); 205 rc++; 206 } 207 if (kvm_riscv_nacl_sync_sret_available()) { 208 if (rc) 209 strcat(slist, ", "); 210 strcat(slist, "sync_sret"); 211 rc++; 212 } 213 if (kvm_riscv_nacl_autoswap_csr_available()) { 214 if (rc) 215 strcat(slist, ", "); 216 strcat(slist, "autoswap_csr"); 217 rc++; 218 } 219 kvm_info("using SBI nested acceleration with %s\n", 220 (rc) ? slist : "no features"); 221 } 222 223 kvm_info("highest G-stage page table mode is %s\n", str); 224 225 kvm_info("VMID %ld bits available\n", kvm_riscv_gstage_vmid_bits()); 226 227 kvm_riscv_setup_vendor_features(); 228 229 kvm_riscv_v_init(); 230 231 kvm_register_perf_callbacks(); 232 233 /* Register CPU PM notifier for CPU idle non-retention states */ 234 if (IS_ENABLED(CONFIG_CPU_PM)) { 235 rc = cpu_pm_register_notifier(&kvm_riscv_cpu_pm_nb); 236 if (rc) { 237 kvm_err("Failed to register CPU PM notifier: %d\n", rc); 238 goto err_teardown; 239 } 240 } 241 242 rc = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE); 243 if (rc) 244 goto err_unregister_cpu_pm; 245 246 if (kvm_riscv_aia_available()) 247 kvm_info("AIA available with %d guest external interrupts\n", 248 atomic_read(&kvm_riscv_aia_nr_hgei)); 249 250 return 0; 251 252 err_unregister_cpu_pm: 253 if (IS_ENABLED(CONFIG_CPU_PM)) 254 cpu_pm_unregister_notifier(&kvm_riscv_cpu_pm_nb); 255 err_teardown: 256 kvm_riscv_teardown(); 257 return rc; 258 } 259 module_init(riscv_kvm_init); 260 261 static void __exit riscv_kvm_exit(void) 262 { 263 kvm_exit(); 264 265 /* Unregister CPU PM notifier */ 266 if (IS_ENABLED(CONFIG_CPU_PM)) 267 cpu_pm_unregister_notifier(&kvm_riscv_cpu_pm_nb); 268 269 kvm_riscv_teardown(); 270 } 271 module_exit(riscv_kvm_exit); 272