1 /*- 2 * Copyright (c) 2009-2012,2016-2024 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/param.h> 30 #include <sys/bus.h> 31 #include <sys/kernel.h> 32 #include <sys/linker.h> 33 #include <sys/lock.h> 34 #include <sys/malloc.h> 35 #include <sys/module.h> 36 #include <sys/mutex.h> 37 #include <sys/sbuf.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 #include <sys/systm.h> 41 #include <sys/proc.h> 42 #include <sys/sched.h> 43 #include <sys/kdb.h> 44 #include <vm/vm.h> 45 #include <vm/pmap.h> 46 47 #include <machine/bus.h> 48 #include <dev/hyperv/vmbus/x86/hyperv_machdep.h> 49 #include <dev/hyperv/vmbus/x86/hyperv_reg.h> 50 #include <dev/hyperv/include/hyperv.h> 51 #include <dev/hyperv/vmbus/hyperv_var.h> 52 #include <dev/hyperv/vmbus/vmbus_reg.h> 53 #include <dev/hyperv/vmbus/vmbus_var.h> 54 #include <dev/hyperv/vmbus/hyperv_common_reg.h> 55 #include "hyperv_mmu.h" 56 57 static inline int fill_gva_list(uint64_t gva_list[], 58 unsigned long start, unsigned long end) 59 { 60 int gva_n = 0; 61 unsigned long cur = start, diff; 62 63 do { 64 diff = end > cur ? end - cur : 0; 65 66 gva_list[gva_n] = cur; 67 /* 68 * Lower 12 bits encode the number of additional 69 * pages to flush (in addition to the 'cur' page). 70 */ 71 if (diff >= HV_TLB_FLUSH_UNIT) { 72 gva_list[gva_n] |= PAGE_MASK; 73 cur += HV_TLB_FLUSH_UNIT; 74 } else if (diff) { 75 gva_list[gva_n] |= (diff - 1) >> PAGE_SHIFT; 76 cur = end; 77 } 78 79 gva_n++; 80 81 } while (cur < end); 82 83 return gva_n; 84 } 85 86 static void 87 hv_tlb_flush_params(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2, 88 enum invl_op_codes op, uint64_t *addr_space, uint64_t *flags, 89 vm_offset_t *start, vm_offset_t *end) 90 { 91 switch (op) { 92 case INVL_OP_TLB: 93 case INVL_OP_TLB_PCID: 94 case INVL_OP_TLB_INVPCID_PTI: 95 case INVL_OP_TLB_INVPCID: 96 *start = *end = 0; 97 break; 98 case INVL_OP_PG: 99 case INVL_OP_PG_INVPCID: 100 case INVL_OP_PG_PCID: 101 *start = addr1; 102 *end = addr1 + PAGE_SIZE; 103 break; 104 default: 105 *start = addr1; 106 *end = addr2; 107 break; 108 } 109 if (pmap == kernel_pmap || *end == 0) { 110 *addr_space = 0; 111 *flags = HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES; 112 } else { 113 *addr_space = pmap->pm_cr3 & ~CR3_PCID_MASK; 114 *flags = 0; 115 } 116 if (*end == 0 && pmap != kernel_pmap) 117 *flags |= HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY; 118 } 119 120 inline int hv_cpumask_to_vpset(struct hv_vpset *vpset, 121 const cpuset_t *cpus, struct vmbus_softc * sc) 122 { 123 int cpu, vcpu, vcpu_bank, vcpu_offset, nr_bank = 1; 124 int max_vcpu_bank = hv_max_vp_index / HV_VCPUS_PER_SPARSE_BANK; 125 126 /* 127 * vpset.valid_bank_mask can represent up to 128 * HV_MAX_SPARSE_VCPU_BANKS banks 129 */ 130 if (max_vcpu_bank >= HV_MAX_SPARSE_VCPU_BANKS) 131 return 0; 132 133 /* 134 * Clear all banks up to the maximum possible bank as hv_tlb_flush_ex 135 * structs are not cleared between calls, we risk flushing unneeded 136 * vCPUs otherwise. 137 */ 138 for (vcpu_bank = 0; vcpu_bank <= max_vcpu_bank; vcpu_bank++) 139 vpset->bank_contents[vcpu_bank] = 0; 140 141 /* 142 * Some banks may end up being empty but this is acceptable. 143 */ 144 CPU_FOREACH_ISSET(cpu, cpus) { 145 vcpu = VMBUS_PCPU_GET(sc, vcpuid, cpu); 146 if (vcpu == -1) 147 return -1; 148 vcpu_bank = vcpu / HV_VCPUS_PER_SPARSE_BANK; 149 vcpu_offset = vcpu % HV_VCPUS_PER_SPARSE_BANK; 150 set_bit(vcpu_offset, (unsigned long *) 151 &vpset->bank_contents[vcpu_bank]); 152 if (vcpu_bank >= nr_bank) 153 nr_bank = vcpu_bank + 1; 154 } 155 vpset->valid_bank_mask = GENMASK_ULL(nr_bank - 1, 0); 156 return nr_bank; 157 } 158 159 160 161 162 void 163 hv_vm_tlb_flush(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2, 164 enum invl_op_codes op, struct vmbus_softc *sc, smp_invl_local_cb_t curcpu_cb) 165 { 166 cpuset_t tmp_mask, mask; 167 struct hyperv_tlb_flush *flush; 168 int cpu, vcpu; 169 int max_gvas, gva_n; 170 uint64_t status = 0; 171 vm_offset_t start, end; 172 173 /* 174 * Hyper-V doesn't handle the invalidating cache. Let system handle it. 175 */ 176 if (op == INVL_OP_CACHE) 177 return smp_targeted_tlb_shootdown_native(pmap, addr1, addr2, 178 curcpu_cb, op); 179 180 flush = *VMBUS_PCPU_PTR(sc, cpu_mem, curcpu); 181 if (flush == NULL) 182 return smp_targeted_tlb_shootdown_native(pmap, addr1, addr2, 183 curcpu_cb, op); 184 /* 185 * It is not necessary to signal other CPUs while booting or 186 * when in the debugger. 187 */ 188 if (__predict_false(kdb_active || KERNEL_PANICKED() || !smp_started)) 189 goto local_cb; 190 191 KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); 192 193 /* 194 * Make a stable copy of the set of CPUs on which the pmap is active. 195 * See if we have to interrupt other CPUs. 196 */ 197 CPU_COPY(pmap_invalidate_cpu_mask(pmap), &tmp_mask); 198 CPU_COPY(pmap_invalidate_cpu_mask(pmap), &mask); 199 CPU_CLR(curcpu, &tmp_mask); 200 if (CPU_EMPTY(&tmp_mask)) 201 goto local_cb; 202 203 /* 204 * Initiator must have interrupts enabled, which prevents 205 * non-invalidation IPIs that take smp_ipi_mtx spinlock, 206 * from deadlocking with us. On the other hand, preemption 207 * must be disabled to pin initiator to the instance of the 208 * pcpu pc_smp_tlb data and scoreboard line. 209 */ 210 KASSERT((read_rflags() & PSL_I) != 0, 211 ("hv_tlb_flush: interrupts disabled")); 212 critical_enter(); 213 flush->processor_mask = 0; 214 215 hv_tlb_flush_params(pmap, addr1, addr2, op, &flush->address_space, 216 &flush->flags, &start, &end); 217 if(CPU_CMP(&mask, &all_cpus) == 0) { 218 flush->flags |= HV_FLUSH_ALL_PROCESSORS; 219 } else { 220 if (CPU_FLS(&mask) < mp_ncpus && CPU_FLS(&mask) >= 64) 221 goto do_ex_hypercall; 222 223 CPU_FOREACH_ISSET(cpu, &mask) { 224 vcpu = VMBUS_PCPU_GET(sc, vcpuid, cpu); 225 if (vcpu >= 64) 226 goto do_ex_hypercall; 227 228 set_bit(vcpu, &flush->processor_mask); 229 } 230 if (!flush->processor_mask ) 231 goto native; 232 } 233 max_gvas = (PAGE_SIZE - sizeof(*flush)) / sizeof(flush->gva_list[0]); 234 if (end == 0 || (end - start) / HV_TLB_FLUSH_UNIT > max_gvas) { 235 status = hypercall_do_md(HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE, 236 (uint64_t)flush, (uint64_t)NULL); 237 } else { 238 gva_n = fill_gva_list(flush->gva_list, start, end); 239 240 status = hv_do_rep_hypercall(HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST, 241 gva_n, 0, (uint64_t)flush, (uint64_t)NULL); 242 } 243 if(status) 244 goto native; 245 sched_unpin(); 246 critical_exit(); 247 return; 248 249 local_cb: 250 critical_enter(); 251 curcpu_cb(pmap, addr1, addr2); 252 sched_unpin(); 253 critical_exit(); 254 return; 255 do_ex_hypercall: 256 status = hv_flush_tlb_others_ex(pmap, addr1, addr2, mask, op, sc); 257 if (status) 258 goto native; 259 sched_unpin(); 260 critical_exit(); 261 return; 262 native: 263 critical_exit(); 264 return smp_targeted_tlb_shootdown_native(pmap, addr1, 265 addr2, curcpu_cb, op); 266 } 267 268 uint64_t 269 hv_flush_tlb_others_ex(pmap_t pmap, vm_offset_t addr1, vm_offset_t addr2, 270 const cpuset_t mask, enum invl_op_codes op, struct vmbus_softc *sc) 271 { 272 int nr_bank = 0, max_gvas, gva_n; 273 struct hv_tlb_flush_ex *flush; 274 if(*VMBUS_PCPU_PTR(sc, cpu_mem, curcpu) == NULL) 275 return EINVAL; 276 flush = *VMBUS_PCPU_PTR(sc, cpu_mem, curcpu); 277 uint64_t status = 0; 278 vm_offset_t start, end; 279 280 if (!(hyperv_recommends & HYPERV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) 281 return EINVAL; 282 283 hv_tlb_flush_params(pmap, addr1, addr2, op, &flush->address_space, 284 &flush->flags, &start, &end); 285 flush->hv_vp_set.valid_bank_mask = 0; 286 287 flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K; 288 nr_bank = hv_cpumask_to_vpset(&flush->hv_vp_set, &mask, sc); 289 if (nr_bank < 0) 290 return EINVAL; 291 292 /* 293 * We can flush not more than max_gvas with one hypercall. Flush the 294 * whole address space if we were asked to do more. 295 */ 296 max_gvas = (PAGE_SIZE - sizeof(*flush) - nr_bank * 297 sizeof(flush->hv_vp_set.bank_contents[0])) / 298 sizeof(flush->hv_vp_set.bank_contents[0]); 299 300 if (end == 0 || ((end - start) / HV_TLB_FLUSH_UNIT) > max_gvas) { 301 status = hv_do_rep_hypercall( 302 HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX, 303 0, nr_bank, (uint64_t)flush, (uint64_t)NULL); 304 } else { 305 gva_n = fill_gva_list(&flush->hv_vp_set.bank_contents[nr_bank], 306 start, end); 307 status = hv_do_rep_hypercall( 308 HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX, 309 gva_n, nr_bank, (uint64_t)flush, (uint64_t)NULL); 310 } 311 return status; 312 } 313