1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2025 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by the University of Cambridge Computer 7 * Laboratory (Department of Computer Science and Technology) under Innovate 8 * UK project 105694, "Digital Security by Design (DSbD) Technology Platform 9 * Prototype". 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/smp.h> 36 #include <sys/kernel.h> 37 #include <sys/malloc.h> 38 #include <sys/lock.h> 39 #include <sys/mutex.h> 40 #include <sys/bus.h> 41 42 #include "riscv.h" 43 #include "vmm_fence.h" 44 45 static bool 46 vmm_fence_dequeue(struct hypctx *hypctx, struct vmm_fence *new_fence) 47 { 48 struct vmm_fence *queue; 49 struct vmm_fence *fence; 50 51 mtx_lock_spin(&hypctx->fence_queue_mtx); 52 queue = hypctx->fence_queue; 53 fence = &queue[hypctx->fence_queue_head]; 54 if (fence->type != VMM_RISCV_FENCE_INVALID) { 55 *new_fence = *fence; 56 fence->type = VMM_RISCV_FENCE_INVALID; 57 hypctx->fence_queue_head = 58 (hypctx->fence_queue_head + 1) % VMM_FENCE_QUEUE_SIZE; 59 } else { 60 mtx_unlock_spin(&hypctx->fence_queue_mtx); 61 return (false); 62 } 63 mtx_unlock_spin(&hypctx->fence_queue_mtx); 64 65 return (true); 66 } 67 68 static bool 69 vmm_fence_enqueue(struct hypctx *hypctx, struct vmm_fence *new_fence) 70 { 71 struct vmm_fence *queue; 72 struct vmm_fence *fence; 73 74 mtx_lock_spin(&hypctx->fence_queue_mtx); 75 queue = hypctx->fence_queue; 76 fence = &queue[hypctx->fence_queue_tail]; 77 if (fence->type == VMM_RISCV_FENCE_INVALID) { 78 *fence = *new_fence; 79 hypctx->fence_queue_tail = 80 (hypctx->fence_queue_tail + 1) % VMM_FENCE_QUEUE_SIZE; 81 } else { 82 mtx_unlock_spin(&hypctx->fence_queue_mtx); 83 return (false); 84 } 85 mtx_unlock_spin(&hypctx->fence_queue_mtx); 86 87 return (true); 88 } 89 90 static void 91 vmm_fence_process_one(struct vmm_fence *fence) 92 { 93 uint64_t va; 94 95 KASSERT(fence->type == VMM_RISCV_FENCE_VMA || 96 fence->type == VMM_RISCV_FENCE_VMA_ASID, 97 ("%s: wrong fence type %d", __func__, fence->type)); 98 99 switch (fence->type) { 100 case VMM_RISCV_FENCE_VMA: 101 for (va = fence->start; va < fence->start + fence->size; 102 va += PAGE_SIZE) 103 sfence_vma_page(va); 104 break; 105 case VMM_RISCV_FENCE_VMA_ASID: 106 if ((fence->start == 0 && fence->size == 0) || 107 fence->size == -1) 108 sfence_vma_asid(fence->asid); 109 else 110 for (va = fence->start; va < fence->start + fence->size; 111 va += PAGE_SIZE) 112 sfence_vma_asid_page(fence->asid, va); 113 break; 114 default: 115 break; 116 } 117 } 118 119 void 120 vmm_fence_process(struct hypctx *hypctx) 121 { 122 struct vmm_fence fence; 123 int pending; 124 125 pending = atomic_readandclear_32(&hypctx->fence_req); 126 127 KASSERT((pending & ~(FENCE_REQ_I | FENCE_REQ_VMA)) == 0, 128 ("wrong fence bit mask")); 129 130 if (pending & FENCE_REQ_I) 131 fence_i(); 132 133 if (pending & FENCE_REQ_VMA) 134 sfence_vma(); 135 136 while (vmm_fence_dequeue(hypctx, &fence) == true) 137 vmm_fence_process_one(&fence); 138 } 139 140 void 141 vmm_fence_add(struct vm *vm, cpuset_t *cpus, struct vmm_fence *fence) 142 { 143 struct hypctx *hypctx; 144 cpuset_t running_cpus; 145 struct vcpu *vcpu; 146 uint16_t maxcpus; 147 int hostcpu; 148 int state; 149 bool enq; 150 int i; 151 152 CPU_ZERO(&running_cpus); 153 154 maxcpus = vm_get_maxcpus(vm); 155 for (i = 0; i < maxcpus; i++) { 156 if (!CPU_ISSET(i, cpus)) 157 continue; 158 vcpu = vm_vcpu(vm, i); 159 hypctx = vcpu_get_cookie(vcpu); 160 161 enq = false; 162 163 /* No need to enqueue fences i and vma global. */ 164 switch (fence->type) { 165 case VMM_RISCV_FENCE_I: 166 atomic_set_32(&hypctx->fence_req, FENCE_REQ_I); 167 break; 168 case VMM_RISCV_FENCE_VMA: 169 if ((fence->start == 0 && fence->size == 0) || 170 fence->size == -1) 171 atomic_set_32(&hypctx->fence_req, 172 FENCE_REQ_VMA); 173 else 174 enq = true; 175 break; 176 case VMM_RISCV_FENCE_VMA_ASID: 177 enq = true; 178 break; 179 default: 180 KASSERT(0, ("%s: wrong fence type %d", __func__, 181 fence->type)); 182 break; 183 } 184 185 /* 186 * Try to enqueue. In case of failure use more conservative 187 * request. 188 */ 189 if (enq) 190 if (vmm_fence_enqueue(hypctx, fence) == false) 191 atomic_set_32(&hypctx->fence_req, 192 FENCE_REQ_VMA); 193 194 mb(); 195 196 state = vcpu_get_state(vcpu, &hostcpu); 197 if (state == VCPU_RUNNING) 198 CPU_SET(hostcpu, &running_cpus); 199 } 200 201 /* 202 * Interrupt other cores. On reception of IPI they will leave guest. 203 * On entry back to the guest they will process fence request. 204 * 205 * If vcpu migrates to another cpu right here, it should process 206 * all fences on entry to the guest as well. 207 */ 208 if (!CPU_EMPTY(&running_cpus)) 209 smp_rendezvous_cpus(running_cpus, NULL, NULL, NULL, NULL); 210 } 211