1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020 - Google LLC 4 * Author: Quentin Perret <qperret@google.com> 5 */ 6 #ifndef __ARM64_KVM_PKVM_H__ 7 #define __ARM64_KVM_PKVM_H__ 8 9 #include <linux/arm_ffa.h> 10 #include <linux/kvm_host.h> 11 #include <linux/memblock.h> 12 #include <linux/scatterlist.h> 13 #include <asm/kvm_pgtable.h> 14 15 /* Maximum number of VMs that can co-exist under pKVM. */ 16 #define KVM_MAX_PVMS 255 17 18 #define HYP_MEMBLOCK_REGIONS 128 19 20 void pkvm_selftests(void); 21 int pkvm_init_host_vm(struct kvm *kvm); 22 int pkvm_create_hyp_vm(struct kvm *kvm); 23 bool pkvm_hyp_vm_is_created(struct kvm *kvm); 24 void pkvm_destroy_hyp_vm(struct kvm *kvm); 25 int pkvm_create_hyp_vcpu(struct kvm_vcpu *vcpu); 26 27 /* 28 * Check whether the specific capability is allowed in pKVM. 29 * 30 * Certain features are allowed only for non-protected VMs in pKVM, which is why 31 * this takes the VM (kvm) as a parameter. 32 */ 33 static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) 34 { 35 switch (ext) { 36 case KVM_CAP_IRQCHIP: 37 case KVM_CAP_ARM_PSCI: 38 case KVM_CAP_ARM_PSCI_0_2: 39 case KVM_CAP_NR_VCPUS: 40 case KVM_CAP_MAX_VCPUS: 41 case KVM_CAP_MAX_VCPU_ID: 42 case KVM_CAP_MSI_DEVID: 43 case KVM_CAP_ARM_VM_IPA_SIZE: 44 case KVM_CAP_ARM_PTRAUTH_ADDRESS: 45 case KVM_CAP_ARM_PTRAUTH_GENERIC: 46 case KVM_CAP_ONE_REG: 47 case KVM_CAP_MP_STATE: 48 case KVM_CAP_VCPU_EVENTS: 49 case KVM_CAP_VCPU_ATTRIBUTES: 50 case KVM_CAP_IMMEDIATE_EXIT: 51 case KVM_CAP_IOEVENTFD: 52 case KVM_CAP_IRQFD_RESAMPLE: 53 case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2: 54 case KVM_CAP_ARM_INJECT_SERROR_ESR: 55 return true; 56 case KVM_CAP_ARM_MTE: 57 case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: 58 case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: 59 case KVM_CAP_PRE_FAULT_MEMORY: 60 return false; 61 default: 62 return !kvm || (kvm->arch.vm_flavor == VM_PKVM); 63 } 64 } 65 66 /* 67 * The vCPU features a protected VM may use: checked by the host at 68 * KVM_ARM_VCPU_INIT, applied by EL2 when the hyp VM is created. 69 */ 70 static inline void kvm_pkvm_vcpu_allowed_features(struct kvm *kvm, 71 unsigned long *allowed) 72 { 73 bitmap_zero(allowed, KVM_VCPU_MAX_FEATURES); 74 75 set_bit(KVM_ARM_VCPU_PSCI_0_2, allowed); 76 77 if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PMU_V3)) 78 set_bit(KVM_ARM_VCPU_PMU_V3, allowed); 79 80 if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_ADDRESS)) 81 set_bit(KVM_ARM_VCPU_PTRAUTH_ADDRESS, allowed); 82 83 if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_PTRAUTH_GENERIC)) 84 set_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, allowed); 85 86 if (kvm_pkvm_ext_allowed(kvm, KVM_CAP_ARM_SVE)) 87 set_bit(KVM_ARM_VCPU_SVE, allowed); 88 } 89 90 /* 91 * Check whether the KVM VM IOCTL is allowed in pKVM. 92 * 93 * Certain features are allowed only for non-protected VMs in pKVM, which is why 94 * this takes the VM (kvm) as a parameter. 95 */ 96 static inline bool kvm_pkvm_ioctl_allowed(struct kvm *kvm, unsigned int ioctl) 97 { 98 long ext; 99 int r; 100 101 r = kvm_get_cap_for_kvm_ioctl(ioctl, &ext); 102 if (r < 0) 103 return false; 104 105 return kvm_pkvm_ext_allowed(kvm, ext); 106 } 107 108 extern struct memblock_region kvm_nvhe_sym(hyp_memory)[]; 109 extern unsigned int kvm_nvhe_sym(hyp_memblock_nr); 110 111 static inline unsigned long 112 hyp_vmemmap_memblock_size(struct memblock_region *reg, size_t vmemmap_entry_size) 113 { 114 unsigned long nr_pages = reg->size >> PAGE_SHIFT; 115 unsigned long start, end; 116 117 start = (reg->base >> PAGE_SHIFT) * vmemmap_entry_size; 118 end = start + nr_pages * vmemmap_entry_size; 119 start = ALIGN_DOWN(start, PAGE_SIZE); 120 end = ALIGN(end, PAGE_SIZE); 121 122 return end - start; 123 } 124 125 static inline unsigned long hyp_vmemmap_pages(size_t vmemmap_entry_size) 126 { 127 unsigned long res = 0, i; 128 129 for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) { 130 res += hyp_vmemmap_memblock_size(&kvm_nvhe_sym(hyp_memory)[i], 131 vmemmap_entry_size); 132 } 133 134 return res >> PAGE_SHIFT; 135 } 136 137 static inline unsigned long hyp_vm_table_pages(void) 138 { 139 return PAGE_ALIGN(KVM_MAX_PVMS * sizeof(void *)) >> PAGE_SHIFT; 140 } 141 142 static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages) 143 { 144 unsigned long total = 0; 145 int i; 146 147 /* Provision the worst case scenario */ 148 for (i = KVM_PGTABLE_FIRST_LEVEL; i <= KVM_PGTABLE_LAST_LEVEL; i++) { 149 nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE); 150 total += nr_pages; 151 } 152 153 return total; 154 } 155 156 static inline unsigned long __hyp_pgtable_total_pages(void) 157 { 158 unsigned long res = 0, i; 159 160 /* Cover all of memory with page-granularity */ 161 for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) { 162 struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i]; 163 res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT); 164 } 165 166 return res; 167 } 168 169 static inline unsigned long hyp_s1_pgtable_pages(void) 170 { 171 unsigned long res; 172 173 res = __hyp_pgtable_total_pages(); 174 175 /* Allow 1 GiB for private mappings */ 176 res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); 177 178 return res; 179 } 180 181 static inline unsigned long host_s2_pgtable_pages(void) 182 { 183 unsigned long res; 184 185 /* 186 * Include an extra 16 pages to safely upper-bound the worst case of 187 * concatenated pgds. 188 */ 189 res = __hyp_pgtable_total_pages() + 16; 190 191 /* Allow 1 GiB for MMIO mappings */ 192 res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); 193 194 return res; 195 } 196 197 #ifdef CONFIG_NVHE_EL2_DEBUG 198 static inline unsigned long pkvm_selftest_pages(void) { return 32; } 199 #else 200 static inline unsigned long pkvm_selftest_pages(void) { return 0; } 201 #endif 202 203 #define KVM_FFA_MBOX_NR_PAGES 1 204 205 static inline unsigned long hyp_ffa_proxy_pages(void) 206 { 207 size_t desc_max; 208 209 /* 210 * The hypervisor FFA proxy needs enough memory to buffer a fragmented 211 * descriptor returned from EL3 in response to a RETRIEVE_REQ call. 212 */ 213 desc_max = sizeof(struct ffa_mem_region) + 214 sizeof(struct ffa_mem_region_attributes) + 215 sizeof(struct ffa_composite_mem_region) + 216 SG_MAX_SEGMENTS * sizeof(struct ffa_mem_region_addr_range); 217 218 /* Plus a page each for the hypervisor's RX and TX mailboxes. */ 219 return (2 * KVM_FFA_MBOX_NR_PAGES) + DIV_ROUND_UP(desc_max, PAGE_SIZE); 220 } 221 222 static inline size_t pkvm_host_sve_state_size(void) 223 { 224 if (!system_supports_sve()) 225 return 0; 226 227 return SVE_SIG_REGS_SIZE(sve_vq_from_vl(kvm_host_sve_max_vl)); 228 } 229 230 struct pkvm_mapping { 231 struct rb_node node; 232 u64 gfn; 233 u64 pfn; 234 struct { 235 u64 nr_pages:48; 236 u64 nc:1; 237 }; 238 u64 __subtree_last; /* Internal member for interval tree */ 239 }; 240 241 enum pkvm_hyp_req_type { 242 PKVM_HYP_NO_REQ = 0, 243 PKVM_HYP_REQ_HYP_ALLOC, 244 PKVM_HYP_REQ_HYP_ALLOC_SELFTEST, 245 __PKVM_HYP_REQ_TYPE_MAX, 246 }; 247 248 #define PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX \ 249 (sizeof(struct arm_smccc_res) - offsetof(struct arm_smccc_res, a2) - 1) 250 251 struct pkvm_hyp_req { 252 u8 type; 253 union { 254 struct { 255 u32 nr_pages; 256 } mem; 257 struct { 258 /* Helper for SMCCC encoding/decoding */ 259 u8 args[PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX]; 260 } args; 261 }; 262 }; 263 264 static inline size_t pkvm_hyp_req_arg_size(u8 type) 265 { 266 struct pkvm_hyp_req *req; 267 268 switch (type) { 269 case PKVM_HYP_NO_REQ: 270 return 0; 271 case PKVM_HYP_REQ_HYP_ALLOC: 272 case PKVM_HYP_REQ_HYP_ALLOC_SELFTEST: 273 return sizeof(req->mem); 274 default: 275 WARN_ON(1); 276 } 277 278 return 0; 279 } 280 281 /* Encode the pending pkvm_hyp_req type into the SMCCC args */ 282 static inline void 283 pkvm_hyp_req_to_smccc(struct kvm_cpu_context *host_ctxt, struct pkvm_hyp_req *req) 284 { 285 u8 *dst, type = req->type; 286 size_t size; 287 288 if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) { 289 host_ctxt->regs.regs[2] = 0; 290 return; 291 } 292 293 size = pkvm_hyp_req_arg_size(type); 294 if (WARN_ON(size > PKVM_HYP_REQ_SMCCC_ARG_SIZE_MAX)) 295 return; 296 297 dst = (u8 *)&host_ctxt->regs.regs[2]; 298 *dst = type; 299 300 memcpy(dst + 1, &req->args, size); 301 } 302 303 /* Return true if a pkvm_hyp_req has been decoded from the SMCCC args */ 304 static inline bool smccc_to_pkvm_hyp_req(struct pkvm_hyp_req *req, struct arm_smccc_res *res) 305 { 306 u8 *src = (u8 *)res + offsetof(struct arm_smccc_res, a2); 307 u8 type = *src; 308 309 if (type == PKVM_HYP_NO_REQ || type >= __PKVM_HYP_REQ_TYPE_MAX) 310 return false; 311 312 req->type = type; 313 memcpy(&req->args, src + 1, pkvm_hyp_req_arg_size(type)); 314 315 return true; 316 } 317 318 int __pkvm_handle_smccc_req(struct arm_smccc_res *res); 319 320 /** 321 * pkvm_call_hyp_req() - Issue an HVC that can return hypervisor requests 322 * @f: Hypervisor function symbol to call. 323 * @...: Arguments to pass to the hypercall. 324 * 325 * Re-issue an HVC and process any pending hypervisor request until completion 326 * or error. 327 * 328 * Only use this helper for HVCs whose hypervisor handlers format their return 329 * registers with pkvm_hyp_req_to_smccc(). 330 * 331 * Return: Result of the hypercall or a negative error if the hyp request 332 * handling failed. 333 */ 334 #define pkvm_call_hyp_req(f, ...) \ 335 ({ \ 336 struct arm_smccc_res __res; \ 337 int __ret; \ 338 do { \ 339 __ret = -1; \ 340 arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(f), ##__VA_ARGS__, &__res); \ 341 if (WARN_ON(__res.a0 != SMCCC_RET_SUCCESS)) \ 342 break; \ 343 __ret = __res.a1; \ 344 if (!__ret) \ 345 break; \ 346 __ret = __pkvm_handle_smccc_req(&__res); \ 347 } while (!__ret); \ 348 __ret; \ 349 }) 350 351 int pkvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm_s2_mmu *mmu, 352 struct kvm_pgtable_mm_ops *mm_ops); 353 void pkvm_pgtable_stage2_destroy_range(struct kvm_pgtable *pgt, 354 u64 addr, u64 size); 355 void pkvm_pgtable_stage2_destroy_pgd(struct kvm_pgtable *pgt); 356 int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, 357 enum kvm_pgtable_prot prot, void *mc, 358 enum kvm_pgtable_walk_flags flags); 359 int pkvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size); 360 int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size); 361 int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size); 362 bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 size, bool mkold); 363 int pkvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, enum kvm_pgtable_prot prot, 364 enum kvm_pgtable_walk_flags flags); 365 void pkvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr, 366 enum kvm_pgtable_walk_flags flags); 367 int pkvm_pgtable_stage2_split(struct kvm_pgtable *pgt, u64 addr, u64 size, 368 struct kvm_mmu_memory_cache *mc); 369 void pkvm_pgtable_stage2_free_unlinked(struct kvm_pgtable_mm_ops *mm_ops, void *pgtable, s8 level); 370 kvm_pte_t *pkvm_pgtable_stage2_create_unlinked(struct kvm_pgtable *pgt, u64 phys, s8 level, 371 enum kvm_pgtable_prot prot, void *mc, 372 bool force_pte); 373 #endif /* __ARM64_KVM_PKVM_H__ */ 374