1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (c) 2024 Ventana Micro Systems Inc. 4 */ 5 6 #ifndef __KVM_NACL_H 7 #define __KVM_NACL_H 8 9 #include <linux/jump_label.h> 10 #include <linux/percpu.h> 11 #include <asm/byteorder.h> 12 #include <asm/csr.h> 13 #include <asm/sbi.h> 14 15 struct kvm_vcpu_arch; 16 17 DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_available); 18 #define kvm_riscv_nacl_available() \ 19 static_branch_unlikely(&kvm_riscv_nacl_available) 20 21 DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_sync_csr_available); 22 #define kvm_riscv_nacl_sync_csr_available() \ 23 static_branch_unlikely(&kvm_riscv_nacl_sync_csr_available) 24 25 DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_sync_hfence_available); 26 #define kvm_riscv_nacl_sync_hfence_available() \ 27 static_branch_unlikely(&kvm_riscv_nacl_sync_hfence_available) 28 29 DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_sync_sret_available); 30 #define kvm_riscv_nacl_sync_sret_available() \ 31 static_branch_unlikely(&kvm_riscv_nacl_sync_sret_available) 32 33 DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_autoswap_csr_available); 34 #define kvm_riscv_nacl_autoswap_csr_available() \ 35 static_branch_unlikely(&kvm_riscv_nacl_autoswap_csr_available) 36 37 struct kvm_riscv_nacl { 38 void *shmem; 39 phys_addr_t shmem_phys; 40 }; 41 DECLARE_PER_CPU(struct kvm_riscv_nacl, kvm_riscv_nacl); 42 43 void __kvm_riscv_nacl_hfence(void *shmem, 44 unsigned long control, 45 unsigned long page_num, 46 unsigned long page_count); 47 48 void __kvm_riscv_nacl_switch_to(struct kvm_vcpu_arch *vcpu_arch, 49 unsigned long sbi_ext_id, 50 unsigned long sbi_func_id); 51 52 int kvm_riscv_nacl_enable(void); 53 54 void kvm_riscv_nacl_disable(void); 55 56 void kvm_riscv_nacl_exit(void); 57 58 int kvm_riscv_nacl_init(void); 59 60 #ifdef CONFIG_32BIT 61 #define lelong_to_cpu(__x) le32_to_cpu(__x) 62 #define cpu_to_lelong(__x) cpu_to_le32(__x) 63 #define __lelong __le32 64 #else 65 #define lelong_to_cpu(__x) le64_to_cpu(__x) 66 #define cpu_to_lelong(__x) cpu_to_le64(__x) 67 #define __lelong __le64 68 #endif 69 70 #define nacl_shmem() \ 71 this_cpu_ptr(&kvm_riscv_nacl)->shmem 72 73 #define nacl_scratch_read_long(__shmem, __offset) \ 74 ({ \ 75 __lelong *__p = (__shmem) + \ 76 SBI_NACL_SHMEM_SCRATCH_OFFSET + \ 77 (__offset); \ 78 lelong_to_cpu(*__p); \ 79 }) 80 81 #define nacl_scratch_write_long(__shmem, __offset, __val) \ 82 do { \ 83 __lelong *__p = (__shmem) + \ 84 SBI_NACL_SHMEM_SCRATCH_OFFSET + \ 85 (__offset); \ 86 *__p = cpu_to_lelong(__val); \ 87 } while (0) 88 89 #define nacl_scratch_write_longs(__shmem, __offset, __array, __count) \ 90 do { \ 91 unsigned int __i; \ 92 __lelong *__p = (__shmem) + \ 93 SBI_NACL_SHMEM_SCRATCH_OFFSET + \ 94 (__offset); \ 95 for (__i = 0; __i < (__count); __i++) \ 96 __p[__i] = cpu_to_lelong((__array)[__i]); \ 97 } while (0) 98 99 #define nacl_sync_hfence(__e) \ 100 sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_SYNC_HFENCE, \ 101 (__e), 0, 0, 0, 0, 0) 102 103 #define nacl_hfence_mkconfig(__type, __order, __vmid, __asid) \ 104 ({ \ 105 unsigned long __c = SBI_NACL_SHMEM_HFENCE_CONFIG_PEND; \ 106 __c |= ((__type) & SBI_NACL_SHMEM_HFENCE_CONFIG_TYPE_MASK) \ 107 << SBI_NACL_SHMEM_HFENCE_CONFIG_TYPE_SHIFT; \ 108 __c |= (((__order) - SBI_NACL_SHMEM_HFENCE_ORDER_BASE) & \ 109 SBI_NACL_SHMEM_HFENCE_CONFIG_ORDER_MASK) \ 110 << SBI_NACL_SHMEM_HFENCE_CONFIG_ORDER_SHIFT; \ 111 __c |= ((__vmid) & SBI_NACL_SHMEM_HFENCE_CONFIG_VMID_MASK) \ 112 << SBI_NACL_SHMEM_HFENCE_CONFIG_VMID_SHIFT; \ 113 __c |= ((__asid) & SBI_NACL_SHMEM_HFENCE_CONFIG_ASID_MASK); \ 114 __c; \ 115 }) 116 117 #define nacl_hfence_mkpnum(__order, __addr) \ 118 ((__addr) >> (__order)) 119 120 #define nacl_hfence_mkpcount(__order, __size) \ 121 ((__size) >> (__order)) 122 123 #define nacl_hfence_gvma(__shmem, __gpa, __gpsz, __order) \ 124 __kvm_riscv_nacl_hfence(__shmem, \ 125 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_GVMA, \ 126 __order, 0, 0), \ 127 nacl_hfence_mkpnum(__order, __gpa), \ 128 nacl_hfence_mkpcount(__order, __gpsz)) 129 130 #define nacl_hfence_gvma_all(__shmem) \ 131 __kvm_riscv_nacl_hfence(__shmem, \ 132 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_GVMA_ALL, \ 133 0, 0, 0), 0, 0) 134 135 #define nacl_hfence_gvma_vmid(__shmem, __vmid, __gpa, __gpsz, __order) \ 136 __kvm_riscv_nacl_hfence(__shmem, \ 137 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_GVMA_VMID, \ 138 __order, __vmid, 0), \ 139 nacl_hfence_mkpnum(__order, __gpa), \ 140 nacl_hfence_mkpcount(__order, __gpsz)) 141 142 #define nacl_hfence_gvma_vmid_all(__shmem, __vmid) \ 143 __kvm_riscv_nacl_hfence(__shmem, \ 144 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_GVMA_VMID_ALL, \ 145 0, __vmid, 0), 0, 0) 146 147 #define nacl_hfence_vvma(__shmem, __vmid, __gva, __gvsz, __order) \ 148 __kvm_riscv_nacl_hfence(__shmem, \ 149 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_VVMA, \ 150 __order, __vmid, 0), \ 151 nacl_hfence_mkpnum(__order, __gva), \ 152 nacl_hfence_mkpcount(__order, __gvsz)) 153 154 #define nacl_hfence_vvma_all(__shmem, __vmid) \ 155 __kvm_riscv_nacl_hfence(__shmem, \ 156 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_VVMA_ALL, \ 157 0, __vmid, 0), 0, 0) 158 159 #define nacl_hfence_vvma_asid(__shmem, __vmid, __asid, __gva, __gvsz, __order)\ 160 __kvm_riscv_nacl_hfence(__shmem, \ 161 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_VVMA_ASID, \ 162 __order, __vmid, __asid), \ 163 nacl_hfence_mkpnum(__order, __gva), \ 164 nacl_hfence_mkpcount(__order, __gvsz)) 165 166 #define nacl_hfence_vvma_asid_all(__shmem, __vmid, __asid) \ 167 __kvm_riscv_nacl_hfence(__shmem, \ 168 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_VVMA_ASID_ALL, \ 169 0, __vmid, __asid), 0, 0) 170 171 #define nacl_csr_read(__shmem, __csr) \ 172 ({ \ 173 __lelong *__a = (__shmem) + SBI_NACL_SHMEM_CSR_OFFSET; \ 174 lelong_to_cpu(__a[SBI_NACL_SHMEM_CSR_INDEX(__csr)]); \ 175 }) 176 177 #define nacl_csr_write(__shmem, __csr, __val) \ 178 do { \ 179 void *__s = (__shmem); \ 180 unsigned int __i = SBI_NACL_SHMEM_CSR_INDEX(__csr); \ 181 __lelong *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ 182 u8 *__b = (__s) + SBI_NACL_SHMEM_DBITMAP_OFFSET; \ 183 __a[__i] = cpu_to_lelong(__val); \ 184 __b[__i >> 3] |= 1U << (__i & 0x7); \ 185 } while (0) 186 187 #define nacl_csr_swap(__shmem, __csr, __val) \ 188 ({ \ 189 void *__s = (__shmem); \ 190 unsigned int __i = SBI_NACL_SHMEM_CSR_INDEX(__csr); \ 191 __lelong *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \ 192 u8 *__b = (__s) + SBI_NACL_SHMEM_DBITMAP_OFFSET; \ 193 unsigned long __r = lelong_to_cpu(__a[__i]); \ 194 __a[__i] = cpu_to_lelong(__val); \ 195 __b[__i >> 3] |= 1U << (__i & 0x7); \ 196 __r; \ 197 }) 198 199 #define nacl_sync_csr(__csr) \ 200 sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_SYNC_CSR, \ 201 (__csr), 0, 0, 0, 0, 0) 202 203 /* 204 * Each ncsr_xyz() macro defined below has it's own static-branch so every 205 * use of ncsr_xyz() macro emits a patchable direct jump. This means multiple 206 * back-to-back ncsr_xyz() macro usage will emit multiple patchable direct 207 * jumps which is sub-optimal. 208 * 209 * Based on the above, it is recommended to avoid multiple back-to-back 210 * ncsr_xyz() macro usage. 211 */ 212 213 #define ncsr_read(__csr) \ 214 ({ \ 215 unsigned long __r; \ 216 if (kvm_riscv_nacl_available()) \ 217 __r = nacl_csr_read(nacl_shmem(), __csr); \ 218 else \ 219 __r = csr_read(__csr); \ 220 __r; \ 221 }) 222 223 #define ncsr_write(__csr, __val) \ 224 do { \ 225 if (kvm_riscv_nacl_sync_csr_available()) \ 226 nacl_csr_write(nacl_shmem(), __csr, __val); \ 227 else \ 228 csr_write(__csr, __val); \ 229 } while (0) 230 231 #define ncsr_swap(__csr, __val) \ 232 ({ \ 233 unsigned long __r; \ 234 if (kvm_riscv_nacl_sync_csr_available()) \ 235 __r = nacl_csr_swap(nacl_shmem(), __csr, __val); \ 236 else \ 237 __r = csr_swap(__csr, __val); \ 238 __r; \ 239 }) 240 241 #define nsync_csr(__csr) \ 242 do { \ 243 if (kvm_riscv_nacl_sync_csr_available()) \ 244 nacl_sync_csr(__csr); \ 245 } while (0) 246 247 #endif 248