1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2016 Joyent, Inc. 14 */ 15 16 #include <sys/comm_page.h> 17 #include <sys/tsc.h> 18 19 20 /* 21 * Interrogate if querying the clock via the comm page is possible. 22 */ 23 int 24 __cp_can_gettime(comm_page_t *cp) 25 { 26 switch (cp->cp_tsc_type) { 27 case TSC_TSCP: 28 case TSC_RDTSC_MFENCE: 29 case TSC_RDTSC_LFENCE: 30 case TSC_RDTSC_CPUID: 31 return (1); 32 default: 33 break; 34 } 35 return (0); 36 } 37 38 #ifdef __amd64 39 40 /* 41 * The functions used for calculating time (both monotonic and wall-clock) are 42 * implemented in assembly on amd64. This is primarily for stack conservation. 43 */ 44 45 #else /* i386 below */ 46 47 /* 48 * ASM-defined functions. 49 */ 50 extern hrtime_t __cp_tsc_read(comm_page_t *); 51 52 /* 53 * These are cloned from TSC and time related code in the kernel. The should 54 * be kept in sync in the case that the source values are changed. 55 */ 56 #define NSEC_SHIFT 5 57 #define ADJ_SHIFT 4 58 #define NANOSEC 1000000000LL 59 60 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) do { \ 61 uint32_t *_l = (uint32_t *)&(tsc); \ 62 uint64_t sc = (uint32_t)(scale); \ 63 (hrt) += (uint64_t)(_l[1] * sc) << NSEC_SHIFT; \ 64 (hrt) += (uint64_t)(_l[0] * sc) >> (32 - NSEC_SHIFT); \ 65 } while (0) 66 67 /* 68 * Userspace version of tsc_gethrtime. 69 * See: uts/i86pc/os/timestamp.c 70 */ 71 hrtime_t 72 __cp_gethrtime(comm_page_t *cp) 73 { 74 uint32_t old_hres_lock; 75 hrtime_t tsc, hrt, tsc_last; 76 77 /* 78 * Several precautions must be taken when collecting the data necessary 79 * to perform an accurate gethrtime calculation. 80 * 81 * While much of the TSC state stored in the comm page is unchanging 82 * after boot, portions of it are periodically updated during OS ticks. 83 * Changes to hres_lock during the course of the copy indicates a 84 * potentially inconsistent snapshot, necessitating a loop. 85 * 86 * Even more complicated is the handling for TSCs which require sync 87 * offsets between different CPUs. Since userspace lacks the luxury of 88 * disabling interrupts, a validation loop checking for CPU migrations 89 * is used. Pathological scheduling could, in theory, "outwit" 90 * this check. Such a possibility is considered an acceptable risk. 91 * 92 */ 93 do { 94 old_hres_lock = cp->cp_hres_lock; 95 tsc_last = cp->cp_tsc_last; 96 hrt = cp->cp_tsc_hrtime_base; 97 tsc = __cp_tsc_read(cp); 98 } while ((old_hres_lock & ~1) != cp->cp_hres_lock); 99 100 if (tsc >= tsc_last) { 101 tsc -= tsc_last; 102 } else if (tsc >= tsc_last - (2 * cp->cp_tsc_max_delta)) { 103 tsc = 0; 104 } else if (tsc > cp->cp_tsc_resume_cap) { 105 tsc = cp->cp_tsc_resume_cap; 106 } 107 TSC_CONVERT_AND_ADD(tsc, hrt, cp->cp_nsec_scale); 108 109 return (hrt); 110 } 111 112 /* 113 * Userspace version of pc_gethrestime. 114 * See: uts/i86pc/os/machdep.c 115 */ 116 int 117 __cp_clock_gettime_realtime(comm_page_t *cp, timespec_t *tsp) 118 { 119 int lock_prev, nslt; 120 timespec_t now; 121 int64_t hres_adj; 122 123 loop: 124 lock_prev = cp->cp_hres_lock; 125 now.tv_sec = cp->cp_hrestime[0]; 126 now.tv_nsec = cp->cp_hrestime[1]; 127 nslt = (int)(__cp_gethrtime(cp) - cp->cp_hres_last_tick); 128 hres_adj = cp->cp_hrestime_adj; 129 if (nslt < 0) { 130 /* 131 * Tick came between sampling hrtime and hres_last_tick; 132 */ 133 goto loop; 134 } 135 now.tv_nsec += nslt; 136 137 /* 138 * Apply hres_adj skew, if needed. 139 */ 140 if (hres_adj > 0) { 141 nslt = (nslt >> ADJ_SHIFT); 142 if (nslt > hres_adj) 143 nslt = (int)hres_adj; 144 now.tv_nsec += nslt; 145 } else if (hres_adj < 0) { 146 nslt = -(nslt >> ADJ_SHIFT); 147 if (nslt < hres_adj) 148 nslt = (int)hres_adj; 149 now.tv_nsec += nslt; 150 } 151 152 /* 153 * Rope in tv_nsec from any excessive adjustments. 154 */ 155 while ((unsigned long)now.tv_nsec >= NANOSEC) { 156 now.tv_nsec -= NANOSEC; 157 now.tv_sec++; 158 } 159 160 if ((cp->cp_hres_lock & ~1) != lock_prev) 161 goto loop; 162 163 *tsp = now; 164 return (0); 165 } 166 167 /* 168 * The __cp_clock_gettime_monotonic function expects that hrt2ts be present 169 * when the code is finally linked. 170 * (The amd64 version has no such requirement.) 171 */ 172 extern void hrt2ts(hrtime_t, timespec_t *); 173 174 int 175 __cp_clock_gettime_monotonic(comm_page_t *cp, timespec_t *tsp) 176 { 177 hrtime_t hrt; 178 179 hrt = __cp_gethrtime(cp); 180 hrt2ts(hrt, tsp); 181 return (0); 182 } 183 184 #endif /* __amd64 */ 185