1*2428aad8SPatrick Mooney /* 2*2428aad8SPatrick Mooney * This file and its contents are supplied under the terms of the 3*2428aad8SPatrick Mooney * Common Development and Distribution License ("CDDL"), version 1.0. 4*2428aad8SPatrick Mooney * You may only use this file in accordance with the terms of version 5*2428aad8SPatrick Mooney * 1.0 of the CDDL. 6*2428aad8SPatrick Mooney * 7*2428aad8SPatrick Mooney * A full copy of the text of the CDDL should have accompanied this 8*2428aad8SPatrick Mooney * source. A copy of the CDDL is also available via the Internet at 9*2428aad8SPatrick Mooney * http://www.illumos.org/license/CDDL. 10*2428aad8SPatrick Mooney */ 11*2428aad8SPatrick Mooney 12*2428aad8SPatrick Mooney /* 13*2428aad8SPatrick Mooney * Copyright 2016 Joyent, Inc. 14*2428aad8SPatrick Mooney */ 15*2428aad8SPatrick Mooney 16*2428aad8SPatrick Mooney #include <sys/comm_page.h> 17*2428aad8SPatrick Mooney #include <sys/tsc.h> 18*2428aad8SPatrick Mooney 19*2428aad8SPatrick Mooney 20*2428aad8SPatrick Mooney /* 21*2428aad8SPatrick Mooney * Interrogate if querying the clock via the comm page is possible. 22*2428aad8SPatrick Mooney */ 23*2428aad8SPatrick Mooney int 24*2428aad8SPatrick Mooney __cp_can_gettime(comm_page_t *cp) 25*2428aad8SPatrick Mooney { 26*2428aad8SPatrick Mooney switch (cp->cp_tsc_type) { 27*2428aad8SPatrick Mooney case TSC_TSCP: 28*2428aad8SPatrick Mooney case TSC_RDTSC_MFENCE: 29*2428aad8SPatrick Mooney case TSC_RDTSC_LFENCE: 30*2428aad8SPatrick Mooney case TSC_RDTSC_CPUID: 31*2428aad8SPatrick Mooney return (1); 32*2428aad8SPatrick Mooney default: 33*2428aad8SPatrick Mooney break; 34*2428aad8SPatrick Mooney } 35*2428aad8SPatrick Mooney return (0); 36*2428aad8SPatrick Mooney } 37*2428aad8SPatrick Mooney 38*2428aad8SPatrick Mooney #ifdef __amd64 39*2428aad8SPatrick Mooney 40*2428aad8SPatrick Mooney /* 41*2428aad8SPatrick Mooney * The functions used for calculating time (both monotonic and wall-clock) are 42*2428aad8SPatrick Mooney * implemented in assembly on amd64. This is primarily for stack conservation. 43*2428aad8SPatrick Mooney */ 44*2428aad8SPatrick Mooney 45*2428aad8SPatrick Mooney #else /* i386 below */ 46*2428aad8SPatrick Mooney 47*2428aad8SPatrick Mooney /* 48*2428aad8SPatrick Mooney * ASM-defined functions. 49*2428aad8SPatrick Mooney */ 50*2428aad8SPatrick Mooney extern hrtime_t __cp_tsc_read(comm_page_t *); 51*2428aad8SPatrick Mooney 52*2428aad8SPatrick Mooney /* 53*2428aad8SPatrick Mooney * These are cloned from TSC and time related code in the kernel. The should 54*2428aad8SPatrick Mooney * be kept in sync in the case that the source values are changed. 55*2428aad8SPatrick Mooney */ 56*2428aad8SPatrick Mooney #define NSEC_SHIFT 5 57*2428aad8SPatrick Mooney #define ADJ_SHIFT 4 58*2428aad8SPatrick Mooney #define NANOSEC 1000000000LL 59*2428aad8SPatrick Mooney 60*2428aad8SPatrick Mooney #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) do { \ 61*2428aad8SPatrick Mooney uint32_t *_l = (uint32_t *)&(tsc); \ 62*2428aad8SPatrick Mooney uint64_t sc = (uint32_t)(scale); \ 63*2428aad8SPatrick Mooney (hrt) += (uint64_t)(_l[1] * sc) << NSEC_SHIFT; \ 64*2428aad8SPatrick Mooney (hrt) += (uint64_t)(_l[0] * sc) >> (32 - NSEC_SHIFT); \ 65*2428aad8SPatrick Mooney } while (0) 66*2428aad8SPatrick Mooney 67*2428aad8SPatrick Mooney /* 68*2428aad8SPatrick Mooney * Userspace version of tsc_gethrtime. 69*2428aad8SPatrick Mooney * See: uts/i86pc/os/timestamp.c 70*2428aad8SPatrick Mooney */ 71*2428aad8SPatrick Mooney hrtime_t 72*2428aad8SPatrick Mooney __cp_gethrtime(comm_page_t *cp) 73*2428aad8SPatrick Mooney { 74*2428aad8SPatrick Mooney uint32_t old_hres_lock; 75*2428aad8SPatrick Mooney hrtime_t tsc, hrt, tsc_last; 76*2428aad8SPatrick Mooney 77*2428aad8SPatrick Mooney /* 78*2428aad8SPatrick Mooney * Several precautions must be taken when collecting the data necessary 79*2428aad8SPatrick Mooney * to perform an accurate gethrtime calculation. 80*2428aad8SPatrick Mooney * 81*2428aad8SPatrick Mooney * While much of the TSC state stored in the comm page is unchanging 82*2428aad8SPatrick Mooney * after boot, portions of it are periodically updated during OS ticks. 83*2428aad8SPatrick Mooney * Changes to hres_lock during the course of the copy indicates a 84*2428aad8SPatrick Mooney * potentially inconsistent snapshot, necessitating a loop. 85*2428aad8SPatrick Mooney * 86*2428aad8SPatrick Mooney * Even more complicated is the handling for TSCs which require sync 87*2428aad8SPatrick Mooney * offsets between different CPUs. Since userspace lacks the luxury of 88*2428aad8SPatrick Mooney * disabling interrupts, a validation loop checking for CPU migrations 89*2428aad8SPatrick Mooney * is used. Pathological scheduling could, in theory, "outwit" 90*2428aad8SPatrick Mooney * this check. Such a possibility is considered an acceptable risk. 91*2428aad8SPatrick Mooney * 92*2428aad8SPatrick Mooney */ 93*2428aad8SPatrick Mooney do { 94*2428aad8SPatrick Mooney old_hres_lock = cp->cp_hres_lock; 95*2428aad8SPatrick Mooney tsc_last = cp->cp_tsc_last; 96*2428aad8SPatrick Mooney hrt = cp->cp_tsc_hrtime_base; 97*2428aad8SPatrick Mooney tsc = __cp_tsc_read(cp); 98*2428aad8SPatrick Mooney } while ((old_hres_lock & ~1) != cp->cp_hres_lock); 99*2428aad8SPatrick Mooney 100*2428aad8SPatrick Mooney if (tsc >= tsc_last) { 101*2428aad8SPatrick Mooney tsc -= tsc_last; 102*2428aad8SPatrick Mooney } else if (tsc >= tsc_last - (2 * cp->cp_tsc_max_delta)) { 103*2428aad8SPatrick Mooney tsc = 0; 104*2428aad8SPatrick Mooney } else if (tsc > cp->cp_tsc_resume_cap) { 105*2428aad8SPatrick Mooney tsc = cp->cp_tsc_resume_cap; 106*2428aad8SPatrick Mooney } 107*2428aad8SPatrick Mooney TSC_CONVERT_AND_ADD(tsc, hrt, cp->cp_nsec_scale); 108*2428aad8SPatrick Mooney 109*2428aad8SPatrick Mooney return (hrt); 110*2428aad8SPatrick Mooney } 111*2428aad8SPatrick Mooney 112*2428aad8SPatrick Mooney /* 113*2428aad8SPatrick Mooney * Userspace version of pc_gethrestime. 114*2428aad8SPatrick Mooney * See: uts/i86pc/os/machdep.c 115*2428aad8SPatrick Mooney */ 116*2428aad8SPatrick Mooney int 117*2428aad8SPatrick Mooney __cp_clock_gettime_realtime(comm_page_t *cp, timespec_t *tsp) 118*2428aad8SPatrick Mooney { 119*2428aad8SPatrick Mooney int lock_prev, nslt; 120*2428aad8SPatrick Mooney timespec_t now; 121*2428aad8SPatrick Mooney int64_t hres_adj; 122*2428aad8SPatrick Mooney 123*2428aad8SPatrick Mooney loop: 124*2428aad8SPatrick Mooney lock_prev = cp->cp_hres_lock; 125*2428aad8SPatrick Mooney now.tv_sec = cp->cp_hrestime[0]; 126*2428aad8SPatrick Mooney now.tv_nsec = cp->cp_hrestime[1]; 127*2428aad8SPatrick Mooney nslt = (int)(__cp_gethrtime(cp) - cp->cp_hres_last_tick); 128*2428aad8SPatrick Mooney hres_adj = cp->cp_hrestime_adj; 129*2428aad8SPatrick Mooney if (nslt < 0) { 130*2428aad8SPatrick Mooney /* 131*2428aad8SPatrick Mooney * Tick came between sampling hrtime and hres_last_tick; 132*2428aad8SPatrick Mooney */ 133*2428aad8SPatrick Mooney goto loop; 134*2428aad8SPatrick Mooney } 135*2428aad8SPatrick Mooney now.tv_nsec += nslt; 136*2428aad8SPatrick Mooney 137*2428aad8SPatrick Mooney /* 138*2428aad8SPatrick Mooney * Apply hres_adj skew, if needed. 139*2428aad8SPatrick Mooney */ 140*2428aad8SPatrick Mooney if (hres_adj > 0) { 141*2428aad8SPatrick Mooney nslt = (nslt >> ADJ_SHIFT); 142*2428aad8SPatrick Mooney if (nslt > hres_adj) 143*2428aad8SPatrick Mooney nslt = (int)hres_adj; 144*2428aad8SPatrick Mooney now.tv_nsec += nslt; 145*2428aad8SPatrick Mooney } else if (hres_adj < 0) { 146*2428aad8SPatrick Mooney nslt = -(nslt >> ADJ_SHIFT); 147*2428aad8SPatrick Mooney if (nslt < hres_adj) 148*2428aad8SPatrick Mooney nslt = (int)hres_adj; 149*2428aad8SPatrick Mooney now.tv_nsec += nslt; 150*2428aad8SPatrick Mooney } 151*2428aad8SPatrick Mooney 152*2428aad8SPatrick Mooney /* 153*2428aad8SPatrick Mooney * Rope in tv_nsec from any excessive adjustments. 154*2428aad8SPatrick Mooney */ 155*2428aad8SPatrick Mooney while ((unsigned long)now.tv_nsec >= NANOSEC) { 156*2428aad8SPatrick Mooney now.tv_nsec -= NANOSEC; 157*2428aad8SPatrick Mooney now.tv_sec++; 158*2428aad8SPatrick Mooney } 159*2428aad8SPatrick Mooney 160*2428aad8SPatrick Mooney if ((cp->cp_hres_lock & ~1) != lock_prev) 161*2428aad8SPatrick Mooney goto loop; 162*2428aad8SPatrick Mooney 163*2428aad8SPatrick Mooney *tsp = now; 164*2428aad8SPatrick Mooney return (0); 165*2428aad8SPatrick Mooney } 166*2428aad8SPatrick Mooney 167*2428aad8SPatrick Mooney /* 168*2428aad8SPatrick Mooney * The __cp_clock_gettime_monotonic function expects that hrt2ts be present 169*2428aad8SPatrick Mooney * when the code is finally linked. 170*2428aad8SPatrick Mooney * (The amd64 version has no such requirement.) 171*2428aad8SPatrick Mooney */ 172*2428aad8SPatrick Mooney extern void hrt2ts(hrtime_t, timespec_t *); 173*2428aad8SPatrick Mooney 174*2428aad8SPatrick Mooney int 175*2428aad8SPatrick Mooney __cp_clock_gettime_monotonic(comm_page_t *cp, timespec_t *tsp) 176*2428aad8SPatrick Mooney { 177*2428aad8SPatrick Mooney hrtime_t hrt; 178*2428aad8SPatrick Mooney 179*2428aad8SPatrick Mooney hrt = __cp_gethrtime(cp); 180*2428aad8SPatrick Mooney hrt2ts(hrt, tsp); 181*2428aad8SPatrick Mooney return (0); 182*2428aad8SPatrick Mooney } 183*2428aad8SPatrick Mooney 184*2428aad8SPatrick Mooney #endif /* __amd64 */ 185