1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * Copyright 2016 Joyent, Inc.
14 */
15
16 #include <sys/comm_page.h>
17 #include <sys/tsc.h>
18
19
20 /*
21 * Interrogate if querying the clock via the comm page is possible.
22 */
23 int
__cp_can_gettime(comm_page_t * cp)24 __cp_can_gettime(comm_page_t *cp)
25 {
26 switch (cp->cp_tsc_type) {
27 case TSC_TSCP:
28 case TSC_RDTSC_MFENCE:
29 case TSC_RDTSC_LFENCE:
30 case TSC_RDTSC_CPUID:
31 return (1);
32 default:
33 break;
34 }
35 return (0);
36 }
37
38 #ifdef __amd64
39
40 /*
41 * The functions used for calculating time (both monotonic and wall-clock) are
42 * implemented in assembly on amd64. This is primarily for stack conservation.
43 */
44
45 #else /* i386 below */
46
47 /*
48 * ASM-defined functions.
49 */
50 extern hrtime_t __cp_tsc_read(comm_page_t *);
51
52 /*
53 * These are cloned from TSC and time related code in the kernel. The should
54 * be kept in sync in the case that the source values are changed.
55 */
56 #define NSEC_SHIFT 5
57 #define ADJ_SHIFT 4
58 #define NANOSEC 1000000000LL
59
60 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) do { \
61 uint32_t *_l = (uint32_t *)&(tsc); \
62 uint64_t sc = (uint32_t)(scale); \
63 (hrt) += (uint64_t)(_l[1] * sc) << NSEC_SHIFT; \
64 (hrt) += (uint64_t)(_l[0] * sc) >> (32 - NSEC_SHIFT); \
65 } while (0)
66
67 /*
68 * Userspace version of tsc_gethrtime.
69 * See: uts/i86pc/os/timestamp.c
70 */
71 hrtime_t
__cp_gethrtime(comm_page_t * cp)72 __cp_gethrtime(comm_page_t *cp)
73 {
74 uint32_t old_hres_lock;
75 hrtime_t tsc, hrt, tsc_last;
76
77 /*
78 * Several precautions must be taken when collecting the data necessary
79 * to perform an accurate gethrtime calculation.
80 *
81 * While much of the TSC state stored in the comm page is unchanging
82 * after boot, portions of it are periodically updated during OS ticks.
83 * Changes to hres_lock during the course of the copy indicates a
84 * potentially inconsistent snapshot, necessitating a loop.
85 *
86 * Even more complicated is the handling for TSCs which require sync
87 * offsets between different CPUs. Since userspace lacks the luxury of
88 * disabling interrupts, a validation loop checking for CPU migrations
89 * is used. Pathological scheduling could, in theory, "outwit"
90 * this check. Such a possibility is considered an acceptable risk.
91 *
92 */
93 do {
94 old_hres_lock = cp->cp_hres_lock;
95 tsc_last = cp->cp_tsc_last;
96 hrt = cp->cp_tsc_hrtime_base;
97 tsc = __cp_tsc_read(cp);
98 } while ((old_hres_lock & ~1) != cp->cp_hres_lock);
99
100 if (tsc >= tsc_last) {
101 tsc -= tsc_last;
102 } else if (tsc >= tsc_last - (2 * cp->cp_tsc_max_delta)) {
103 tsc = 0;
104 } else if (tsc > cp->cp_tsc_resume_cap) {
105 tsc = cp->cp_tsc_resume_cap;
106 }
107 TSC_CONVERT_AND_ADD(tsc, hrt, cp->cp_nsec_scale);
108
109 return (hrt);
110 }
111
112 /*
113 * Userspace version of pc_gethrestime.
114 * See: uts/i86pc/os/machdep.c
115 */
116 int
__cp_clock_gettime_realtime(comm_page_t * cp,timespec_t * tsp)117 __cp_clock_gettime_realtime(comm_page_t *cp, timespec_t *tsp)
118 {
119 int lock_prev, nslt;
120 timespec_t now;
121 int64_t hres_adj;
122
123 loop:
124 lock_prev = cp->cp_hres_lock;
125 now.tv_sec = cp->cp_hrestime[0];
126 now.tv_nsec = cp->cp_hrestime[1];
127 nslt = (int)(__cp_gethrtime(cp) - cp->cp_hres_last_tick);
128 hres_adj = cp->cp_hrestime_adj;
129 if (nslt < 0) {
130 /*
131 * Tick came between sampling hrtime and hres_last_tick;
132 */
133 goto loop;
134 }
135 now.tv_nsec += nslt;
136
137 /*
138 * Apply hres_adj skew, if needed.
139 */
140 if (hres_adj > 0) {
141 nslt = (nslt >> ADJ_SHIFT);
142 if (nslt > hres_adj)
143 nslt = (int)hres_adj;
144 now.tv_nsec += nslt;
145 } else if (hres_adj < 0) {
146 nslt = -(nslt >> ADJ_SHIFT);
147 if (nslt < hres_adj)
148 nslt = (int)hres_adj;
149 now.tv_nsec += nslt;
150 }
151
152 /*
153 * Rope in tv_nsec from any excessive adjustments.
154 */
155 while ((unsigned long)now.tv_nsec >= NANOSEC) {
156 now.tv_nsec -= NANOSEC;
157 now.tv_sec++;
158 }
159
160 if ((cp->cp_hres_lock & ~1) != lock_prev)
161 goto loop;
162
163 *tsp = now;
164 return (0);
165 }
166
167 /*
168 * The __cp_clock_gettime_monotonic function expects that hrt2ts be present
169 * when the code is finally linked.
170 * (The amd64 version has no such requirement.)
171 */
172 extern void hrt2ts(hrtime_t, timespec_t *);
173
174 int
__cp_clock_gettime_monotonic(comm_page_t * cp,timespec_t * tsp)175 __cp_clock_gettime_monotonic(comm_page_t *cp, timespec_t *tsp)
176 {
177 hrtime_t hrt;
178
179 hrt = __cp_gethrtime(cp);
180 hrt2ts(hrt, tsp);
181 return (0);
182 }
183
184 #endif /* __amd64 */
185