/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #pragma ident "%Z%%M% %I% %E% SMI" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Using the Pentium's TSC register for gethrtime() * ------------------------------------------------ * * The Pentium family, like many chip architectures, has a high-resolution * timestamp counter ("TSC") which increments once per CPU cycle. The contents * of the timestamp counter are read with the RDTSC instruction. * * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count * must be translated into nanoseconds in order to implement gethrtime(). * We avoid inducing floating point operations in this conversion by * implementing the same nsec_scale algorithm as that found in the sun4u * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains * a detailed description of the algorithm; the comment is not reproduced * here. This implementation differs only in its value for NSEC_SHIFT: * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for * 60 MHz Pentiums. * * While TSC and %tick are both cycle counting registers, TSC's functionality * falls short in several critical ways: * * (a) TSCs on different CPUs are not guaranteed to be in sync. While in * practice they often _are_ in sync, this isn't guaranteed by the * architecture. * * (b) The TSC cannot be reliably set to an arbitrary value. The architecture * only supports writing the low 32-bits of TSC, making it impractical * to rewrite. * * (c) The architecture doesn't have the capacity to interrupt based on * arbitrary values of TSC; there is no TICK_CMPR equivalent. * * Together, (a) and (b) imply that software must track the skew between * TSCs and account for it (it is assumed that while there may exist skew, * there does not exist drift). To determine the skew between CPUs, we * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing * the online operation calls tsc_sync_master(). Once both CPUs are ready, * the master sets a shared flag, and each reads its TSC register. To reduce * bias, we then wait until both CPUs are ready again, but this time the * slave sets the shared flag, and each reads its TSC register again. The * master compares the average of the two sample values, and, if observable * skew is found, changes the gethrtimef function pointer to point to a * gethrtime() implementation which will take the discovered skew into * consideration. * * In the absence of time-of-day clock adjustments, gethrtime() must stay in * sync with gettimeofday(). This is problematic; given (c), the software * cannot drive its time-of-day source from TSC, and yet they must somehow be * kept in sync. We implement this by having a routine, tsc_tick(), which * is called once per second from the interrupt which drives time-of-day. * tsc_tick() recalculates nsec_scale based on the number of the CPU cycles * since boot versus the number of seconds since boot. This algorithm * becomes more accurate over time and converges quickly; the error in * nsec_scale is typically under 1 ppm less than 10 seconds after boot, and * is less than 100 ppb 1 minute after boot. * * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified * atomically with nsec_scale under CLOCK_LOCK. This assures that time * monotonically increases. */ #define NSEC_SHIFT 5 static uint_t nsec_scale; /* * These two variables used to be grouped together inside of a structure that * lived on a single cache line. A regression (bug ID 4623398) caused the * compiler to emit code that "optimized" away the while-loops below. The * result was that no synchronization between the onlining and onlined CPUs * took place. */ static volatile int tsc_ready; static volatile int tsc_sync_go; /* * Used as indices into the tsc_sync_snaps[] array. */ #define TSC_MASTER 0 #define TSC_SLAVE 1 /* * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. */ #define TSC_SYNC_STOP 1 #define TSC_SYNC_GO 2 #define TSC_SYNC_AGAIN 3 /* * XX64 Is the faster way to do this with a 64-bit ABI? */ #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ unsigned int *_l = (unsigned int *)&(tsc); \ (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ } #define TSC_CONVERT(tsc, hrt, scale) { \ unsigned int *_l = (unsigned int *)&(tsc); \ (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ } int tsc_master_slave_sync_needed = 1; static int tsc_max_delta; static hrtime_t tsc_sync_snaps[2]; static hrtime_t tsc_sync_delta[NCPU]; static hrtime_t tsc_sync_tick_delta[NCPU]; static hrtime_t tsc_last = 0; static hrtime_t tsc_last_jumped = 0; static hrtime_t tsc_hrtime_base = 0; static int tsc_jumped = 0; static hrtime_t shadow_tsc_hrtime_base; static hrtime_t shadow_tsc_last; static uint_t shadow_nsec_scale; static uint32_t shadow_hres_lock; /* * Called by the master after the sync operation is complete. If the * slave is discovered to lag, gethrtimef will be changed to point to * tsc_gethrtime_delta(). */ static void tsc_digest(processorid_t target) { hrtime_t tdelta, hdelta = 0; int max = tsc_max_delta; processorid_t source = CPU->cpu_id; int update; update = tsc_sync_delta[source] != 0 || gethrtimef == tsc_gethrtime_delta; /* * We divide by 2 since each of the data points is the sum of two TSC * reads; this takes the average of the two. */ tdelta = (tsc_sync_snaps[TSC_SLAVE] - tsc_sync_snaps[TSC_MASTER]) / 2; if ((tdelta > max) || ((tdelta >= 0) && update)) { TSC_CONVERT_AND_ADD(tdelta, hdelta, nsec_scale); tsc_sync_delta[target] = tsc_sync_delta[source] - hdelta; tsc_sync_tick_delta[target] = -tdelta; gethrtimef = tsc_gethrtime_delta; gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; return; } tdelta = -tdelta; if ((tdelta > max) || update) { TSC_CONVERT_AND_ADD(tdelta, hdelta, nsec_scale); tsc_sync_delta[target] = tsc_sync_delta[source] + hdelta; tsc_sync_tick_delta[target] = tdelta; gethrtimef = tsc_gethrtime_delta; gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; } } /* * Called by a CPU which has just performed an online operation on another * CPU. It is expected that the newly onlined CPU will call tsc_sync_slave(). */ void tsc_sync_master(processorid_t slave) { ulong_t flags; hrtime_t hrt; if (!tsc_master_slave_sync_needed) return; ASSERT(tsc_sync_go != TSC_SYNC_GO); flags = clear_int_flag(); /* * Wait for the slave CPU to arrive. */ while (tsc_ready != TSC_SYNC_GO) continue; /* * Tell the slave CPU to begin reading its TSC; read our own. */ tsc_sync_go = TSC_SYNC_GO; hrt = tsc_read(); /* * Tell the slave that we're ready, and wait for the slave to tell us * to read our TSC again. */ tsc_ready = TSC_SYNC_AGAIN; while (tsc_sync_go != TSC_SYNC_AGAIN) continue; hrt += tsc_read(); tsc_sync_snaps[TSC_MASTER] = hrt; /* * Wait for the slave to finish reading its TSC. */ while (tsc_ready != TSC_SYNC_STOP) continue; /* * At this point, both CPUs have performed their tsc_read() calls. * We'll digest it now before letting the slave CPU return. */ tsc_digest(slave); tsc_sync_go = TSC_SYNC_STOP; restore_int_flag(flags); } /* * Called by a CPU which has just been onlined. It is expected that the CPU * performing the online operation will call tsc_sync_master(). */ void tsc_sync_slave(void) { ulong_t flags; hrtime_t hrt; if (!tsc_master_slave_sync_needed) return; ASSERT(tsc_sync_go != TSC_SYNC_GO); flags = clear_int_flag(); /* to test tsc_gethrtime_delta, add wrmsr(REG_TSC, 0) here */ /* * Tell the master CPU that we're ready, and wait for the master to * tell us to begin reading our TSC. */ tsc_ready = TSC_SYNC_GO; while (tsc_sync_go != TSC_SYNC_GO) continue; hrt = tsc_read(); /* * Wait for the master CPU to be ready to read its TSC again. */ while (tsc_ready != TSC_SYNC_AGAIN) continue; /* * Tell the master CPU to read its TSC again; read ours again. */ tsc_sync_go = TSC_SYNC_AGAIN; hrt += tsc_read(); tsc_sync_snaps[TSC_SLAVE] = hrt; /* * Tell the master that we're done, and wait to be dismissed. */ tsc_ready = TSC_SYNC_STOP; while (tsc_sync_go != TSC_SYNC_STOP) continue; restore_int_flag(flags); } void tsc_hrtimeinit(uint64_t cpu_freq_hz) { longlong_t tsc; ulong_t flags; /* * cpu_freq_hz is the measured cpu frequency in hertz */ /* * We can't accommodate CPUs slower than 31.25 MHz. */ ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); nsec_scale = (uint_t) (((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); flags = clear_int_flag(); tsc = tsc_read(); (void) tsc_gethrtime(); tsc_max_delta = tsc_read() - tsc; restore_int_flag(flags); } /* * Called once per second on a CPU from the cyclic subsystem's * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) */ void tsc_tick(void) { hrtime_t now, delta; ushort_t spl; /* * Before we set the new variables, we set the shadow values. This * allows for lock free operation in dtrace_gethrtime(). */ lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, ipltospl(CBE_HIGH_PIL), &spl); shadow_tsc_hrtime_base = tsc_hrtime_base; shadow_tsc_last = tsc_last; shadow_nsec_scale = nsec_scale; shadow_hres_lock++; splx(spl); CLOCK_LOCK(&spl); now = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) now += tsc_sync_tick_delta[CPU->cpu_id]; if (now < tsc_last) { /* * The TSC has just jumped into the past. We assume that * this is due to a suspend/resume cycle, and we're going * to use the _current_ value of TSC as the delta. This * will keep tsc_hrtime_base correct. We're also going to * assume that rate of tsc does not change after a suspend * resume (i.e nsec_scale remains the same). */ delta = now; tsc_last_jumped += tsc_last; tsc_jumped = 1; } else { /* * Determine the number of TSC ticks since the last clock * tick, and add that to the hrtime base. */ delta = now - tsc_last; } TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); tsc_last = now; CLOCK_UNLOCK(spl); } hrtime_t tsc_gethrtime(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; do { old_hres_lock = hres_lock; if ((tsc = tsc_read()) >= tsc_last) { /* * It would seem to be obvious that this is true * (that is, the past is less than the present), * but it isn't true in the presence of suspend/resume * cycles. If we manage to call gethrtime() * after a resume, but before the first call to * tsc_tick(), we will see the jump. In this case, * we will simply use the value in TSC as the delta. */ tsc -= tsc_last; } else if (tsc >= tsc_last - 2*tsc_max_delta) { /* * There is a chance that tsc_tick() has just run on * another CPU, and we have drifted just enough so that * we appear behind tsc_last. In this case, force the * delta to be zero. */ tsc = 0; } hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); } while ((old_hres_lock & ~1) != hres_lock); return (hrt); } /* * This is similar to the above, but it cannot actually spin on hres_lock. * As a result, it caches all of the variables it needs; if the variables * don't change, it's done. */ hrtime_t dtrace_gethrtime(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; int flags; do { old_hres_lock = hres_lock; /* * Interrupts are disabled to ensure that the thread isn't * migrated between the tsc_read() and adding the CPU's * TSC tick delta. */ flags = clear_int_flag(); tsc = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) tsc += tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* * See the comments in tsc_gethrtime(), above. */ if (tsc >= tsc_last) tsc -= tsc_last; else if (tsc >= tsc_last - 2*tsc_max_delta) tsc = 0; hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); if ((old_hres_lock & ~1) == hres_lock) break; /* * If we're here, the clock lock is locked -- or it has been * unlocked and locked since we looked. This may be due to * tsc_tick() running on another CPU -- or it may be because * some code path has ended up in dtrace_probe() with * CLOCK_LOCK held. We'll try to determine that we're in * the former case by taking another lap if the lock has * changed since when we first looked at it. */ if (old_hres_lock != hres_lock) continue; /* * So the lock was and is locked. We'll use the old data * instead. */ old_hres_lock = shadow_hres_lock; /* * Again, disable interrupts to ensure that the thread * isn't migrated between the tsc_read() and adding * the CPU's TSC tick delta. */ flags = clear_int_flag(); tsc = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) tsc += tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* * See the comments in tsc_gethrtime(), above. */ if (tsc >= shadow_tsc_last) tsc -= shadow_tsc_last; else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) tsc = 0; hrt = shadow_tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); } while ((old_hres_lock & ~1) != shadow_hres_lock); return (hrt); } hrtime_t tsc_gethrtime_delta(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; int flags; do { old_hres_lock = hres_lock; /* * We need to disable interrupts here to assure that we * don't migrate between the call to tsc_read() and * adding the CPU's TSC tick delta. Note that disabling * and reenabling preemption is forbidden here because * we may be in the middle of a fast trap. In the amd64 * kernel we cannot tolerate preemption during a fast * trap. See _update_sregs(). */ flags = clear_int_flag(); tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* See comments in tsc_gethrtime() above */ if (tsc >= tsc_last) { tsc -= tsc_last; } else if (tsc >= tsc_last - 2 * tsc_max_delta) { tsc = 0; } hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); } while ((old_hres_lock & ~1) != hres_lock); return (hrt); } extern uint64_t cpu_freq_hz; extern int tsc_gethrtime_enable; /* * The following converts nanoseconds of highres-time to ticks */ static uint64_t hrtime2tick(hrtime_t ts) { hrtime_t q = ts / NANOSEC; hrtime_t r = ts - (q * NANOSEC); return (q * cpu_freq_hz + ((r * cpu_freq_hz) / NANOSEC)); } /* * This is used to convert scaled high-res time from nanoseconds to * unscaled hardware ticks. (Read from hardware timestamp counter) */ uint64_t unscalehrtime(hrtime_t ts) { if (tsc_gethrtime_enable) { uint64_t unscale = 0; hrtime_t rescale; hrtime_t diff = ts; while (diff > (nsec_per_tick)) { unscale += hrtime2tick(diff); rescale = unscale; scalehrtime(&rescale); diff = ts - rescale; } return (unscale); } return (0); } hrtime_t tsc_gethrtimeunscaled(void) { uint32_t old_hres_lock; hrtime_t tsc; do { old_hres_lock = hres_lock; /* See tsc_tick(). */ tsc = tsc_read() + tsc_last_jumped; } while ((old_hres_lock & ~1) != hres_lock); return (tsc); } /* Convert a tsc timestamp to nanoseconds */ void tsc_scalehrtime(hrtime_t *tsc) { hrtime_t hrt; hrtime_t mytsc; if (tsc == NULL) return; mytsc = *tsc; TSC_CONVERT(mytsc, hrt, nsec_scale); *tsc = hrt; } hrtime_t tsc_gethrtimeunscaled_delta(void) { hrtime_t hrt; int flags; /* * Similarly to tsc_gethrtime_delta, we need to disable preemption * to prevent migration between the call to tsc_gethrtimeunscaled * and adding the CPU's hrtime delta. Note that disabling and * reenabling preemption is forbidden here because we may be in the * middle of a fast trap. In the amd64 kernel we cannot tolerate * preemption during a fast trap. See _update_sregs(). */ flags = clear_int_flag(); hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); return (hrt); }