1 /*- 2 * Copyright (c) 2016-2017 Microsoft Corp. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/conf.h> 32 #include <sys/fcntl.h> 33 #include <sys/kernel.h> 34 #include <sys/malloc.h> 35 #include <sys/systm.h> 36 #include <sys/timetc.h> 37 #include <sys/vdso.h> 38 39 #include <machine/cpufunc.h> 40 #include <machine/cputypes.h> 41 #include <machine/md_var.h> 42 #include <machine/specialreg.h> 43 44 #include <vm/vm.h> 45 #include <vm/vm_extern.h> 46 #include <vm/pmap.h> 47 48 #include <dev/hyperv/include/hyperv.h> 49 #include <dev/hyperv/include/hyperv_busdma.h> 50 #include <dev/hyperv/vmbus/x86/hyperv_machdep.h> 51 #include <dev/hyperv/vmbus/x86/hyperv_reg.h> 52 #include <dev/hyperv/vmbus/hyperv_var.h> 53 #include <dev/hyperv/vmbus/hyperv_common_reg.h> 54 55 struct hyperv_reftsc_ctx { 56 struct hyperv_reftsc *tsc_ref; 57 }; 58 59 static uint32_t hyperv_tsc_vdso_timehands( 60 struct vdso_timehands *, 61 struct timecounter *); 62 63 static d_open_t hyperv_tsc_open; 64 static d_mmap_t hyperv_tsc_mmap; 65 66 static struct timecounter hyperv_tsc_timecounter = { 67 .tc_get_timecount = NULL, /* based on CPU vendor. */ 68 .tc_counter_mask = 0xffffffff, 69 .tc_frequency = HYPERV_TIMER_FREQ, 70 .tc_name = "Hyper-V-TSC", 71 .tc_quality = 3000, 72 .tc_fill_vdso_timehands = hyperv_tsc_vdso_timehands, 73 }; 74 75 static struct cdevsw hyperv_tsc_cdevsw = { 76 .d_version = D_VERSION, 77 .d_open = hyperv_tsc_open, 78 .d_mmap = hyperv_tsc_mmap, 79 .d_name = HYPERV_REFTSC_DEVNAME 80 }; 81 82 static struct hyperv_reftsc_ctx hyperv_ref_tsc; 83 84 uint64_t 85 hypercall_md(volatile void *hc_addr, uint64_t in_val, 86 uint64_t in_paddr, uint64_t out_paddr) 87 { 88 uint64_t status; 89 90 __asm__ __volatile__ ("mov %0, %%r8" : : "r" (out_paddr): "r8"); 91 __asm__ __volatile__ ("call *%3" : "=a" (status) : 92 "c" (in_val), "d" (in_paddr), "m" (hc_addr)); 93 return (status); 94 } 95 96 static int 97 hyperv_tsc_open(struct cdev *dev __unused, int oflags, int devtype __unused, 98 struct thread *td __unused) 99 { 100 101 if (oflags & FWRITE) 102 return (EPERM); 103 return (0); 104 } 105 106 static int 107 hyperv_tsc_mmap(struct cdev *dev __unused, vm_ooffset_t offset, 108 vm_paddr_t *paddr, int nprot __unused, vm_memattr_t *memattr __unused) 109 { 110 111 KASSERT(hyperv_ref_tsc.tsc_ref != NULL, ("reftsc has not been setup")); 112 113 /* 114 * NOTE: 115 * 'nprot' does not contain information interested to us; 116 * WR-open is blocked by d_open. 117 */ 118 119 if (offset != 0) 120 return (EOPNOTSUPP); 121 122 *paddr = pmap_kextract((vm_offset_t)hyperv_ref_tsc.tsc_ref); 123 return (0); 124 } 125 126 static uint32_t 127 hyperv_tsc_vdso_timehands(struct vdso_timehands *vdso_th, 128 struct timecounter *tc __unused) 129 { 130 131 vdso_th->th_algo = VDSO_TH_ALGO_X86_HVTSC; 132 vdso_th->th_x86_shift = 0; 133 vdso_th->th_x86_hpet_idx = 0; 134 vdso_th->th_x86_pvc_last_systime = 0; 135 vdso_th->th_x86_pvc_stable_mask = 0; 136 bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); 137 return (1); 138 } 139 140 #define HYPERV_TSC_TIMECOUNT(fence) \ 141 static uint64_t \ 142 hyperv_tc64_tsc_##fence(void) \ 143 { \ 144 struct hyperv_reftsc *tsc_ref = hyperv_ref_tsc.tsc_ref; \ 145 uint32_t seq; \ 146 \ 147 while ((seq = atomic_load_acq_int(&tsc_ref->tsc_seq)) != 0) { \ 148 uint64_t disc, ret, tsc; \ 149 uint64_t scale = tsc_ref->tsc_scale; \ 150 int64_t ofs = tsc_ref->tsc_ofs; \ 151 \ 152 fence(); \ 153 tsc = rdtsc(); \ 154 \ 155 /* ret = ((tsc * scale) >> 64) + ofs */ \ 156 __asm__ __volatile__ ("mulq %3" : \ 157 "=d" (ret), "=a" (disc) : \ 158 "a" (tsc), "r" (scale)); \ 159 ret += ofs; \ 160 \ 161 atomic_thread_fence_acq(); \ 162 if (tsc_ref->tsc_seq == seq) \ 163 return (ret); \ 164 \ 165 /* Sequence changed; re-sync. */ \ 166 } \ 167 /* Fallback to the generic timecounter, i.e. rdmsr. */ \ 168 return (rdmsr(MSR_HV_TIME_REF_COUNT)); \ 169 } \ 170 \ 171 static u_int \ 172 hyperv_tsc_timecount_##fence(struct timecounter *tc __unused) \ 173 { \ 174 \ 175 return (hyperv_tc64_tsc_##fence()); \ 176 } \ 177 struct __hack 178 179 HYPERV_TSC_TIMECOUNT(lfence); 180 HYPERV_TSC_TIMECOUNT(mfence); 181 182 static void 183 hyperv_tsc_tcinit(void *dummy __unused) 184 { 185 hyperv_tc64_t tc64 = NULL; 186 uint64_t val, orig; 187 188 if ((hyperv_features & 189 (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC)) != 190 (CPUID_HV_MSR_TIME_REFCNT | CPUID_HV_MSR_REFERENCE_TSC) || 191 (cpu_feature & CPUID_SSE2) == 0) /* SSE2 for mfence/lfence */ 192 return; 193 194 switch (cpu_vendor_id) { 195 case CPU_VENDOR_AMD: 196 case CPU_VENDOR_HYGON: 197 hyperv_tsc_timecounter.tc_get_timecount = 198 hyperv_tsc_timecount_mfence; 199 tc64 = hyperv_tc64_tsc_mfence; 200 break; 201 202 case CPU_VENDOR_INTEL: 203 hyperv_tsc_timecounter.tc_get_timecount = 204 hyperv_tsc_timecount_lfence; 205 tc64 = hyperv_tc64_tsc_lfence; 206 break; 207 208 default: 209 /* Unsupported CPU vendors. */ 210 return; 211 } 212 213 hyperv_ref_tsc.tsc_ref = contigmalloc(PAGE_SIZE, M_DEVBUF, 214 M_WAITOK | M_ZERO, 0ul, ~0ul, PAGE_SIZE, 0); 215 if (hyperv_ref_tsc.tsc_ref == NULL) { 216 printf("hyperv: reftsc page allocation failed\n"); 217 return; 218 } 219 220 orig = rdmsr(MSR_HV_REFERENCE_TSC); 221 val = (pmap_kextract((vm_offset_t)hyperv_ref_tsc.tsc_ref) >> 222 PAGE_SHIFT) << MSR_HV_REFTSC_PGSHIFT; 223 val |= MSR_HV_REFTSC_ENABLE | (orig & MSR_HV_REFTSC_RSVD_MASK); 224 wrmsr(MSR_HV_REFERENCE_TSC, val); 225 226 /* Register "enlightened" timecounter. */ 227 tc_init(&hyperv_tsc_timecounter); 228 229 /* Install 64 bits timecounter method for other modules to use. */ 230 KASSERT(tc64 != NULL, ("tc64 is not set")); 231 hyperv_tc64 = tc64; 232 233 /* Add device for mmap(2). */ 234 make_dev(&hyperv_tsc_cdevsw, 0, UID_ROOT, GID_WHEEL, 0444, 235 HYPERV_REFTSC_DEVNAME); 236 } 237 SYSINIT(hyperv_tsc_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, hyperv_tsc_tcinit, 238 NULL); 239