1/* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12/* 13 * Copyright 2016 Joyent, Inc. 14 */ 15 16#include <sys/asm_linkage.h> 17#include <sys/segments.h> 18#include <sys/time_impl.h> 19#include <sys/tsc.h> 20#include <cp_offsets.h> 21 22#define GETCPU_GDT_OFFSET SEL_GDT(GDT_CPUID, SEL_UPL) 23 24 .file "cp_subr.s" 25 26/* 27 * hrtime_t 28 * __cp_tsc_read(uint_t cp_tsc_type) 29 * 30 * Stack usage: 0x18 bytes 31 */ 32 ENTRY_NP(__cp_tsc_read) 33 pushl %ebp 34 movl %esp, %ebp 35 pushl %edi 36 pushl %esi 37 subl $0x4, %esp 38 39 movl 0x8(%ebp), %edi 40 movl CP_TSC_TYPE(%edi), %eax 41 movl CP_TSC_NCPU(%edi), %esi 42 cmpl $TSC_TSCP, %eax 43 jne 3f 44 rdtscp 45 cmpl $0, %esi 46 jne 2f 471: 48 addl $0x4, %esp 49 popl %esi 50 popl %edi 51 leave 52 ret 532: 54 /* 55 * When cp_tsc_ncpu is non-zero, it indicates the length of the 56 * cp_tsc_sync_tick_delta array, which contains per-CPU offsets for the 57 * TSC. The CPU ID furnished by the IA32_TSC_AUX register via rdtscp 58 * is used to look up an offset value in that array and apply it to the 59 * TSC reading. 60 */ 61 leal CP_TSC_SYNC_TICK_DELTA(%edi), %esi 62 leal (%esi, %ecx, 8), %ecx 63 addl (%ecx), %eax 64 adcl 0x4(%ecx), %edx 65 jmp 1b 66 673: 68 cmpl $0, %esi 69 je 4f 70 mov $GETCPU_GDT_OFFSET, %eax 71 lsl %ax, %eax 72 movl %eax, (%esp) 73 movl CP_TSC_TYPE(%edi), %eax 74 754: 76 cmpl $TSC_RDTSC_MFENCE, %eax 77 jne 5f 78 mfence 79 rdtsc 80 jmp 8f 81 825: 83 cmpl $TSC_RDTSC_LFENCE, %eax 84 jne 6f 85 lfence 86 rdtsc 87 jmp 8f 88 896: 90 cmpl $TSC_RDTSC_CPUID, %eax 91 jne 7f 92 pushl %ebx 93 xorl %eax, %eax 94 cpuid 95 rdtsc 96 popl %ebx 97 jmp 8f 98 997: 100 /* 101 * Other protections should have prevented this function from being 102 * called in the first place. The only sane action is to abort. 103 * The easiest means in this context is via SIGILL. 104 */ 105 ud2a 106 1078: 108 109 cmpl $0, %esi 110 je 1b 111 /* 112 * With a TSC reading in-hand, confirm that the thread has not migrated 113 * since the cpu_id was first checked. 114 */ 115 movl $GETCPU_GDT_OFFSET, %ecx 116 lsl %cx, %ecx 117 movl (%esp), %esi 118 cmpl %ecx, %esi 119 je 9f 120 /* 121 * There was a CPU migration, perform another reading. 122 */ 123 movl %eax, (%esp) 124 movl CP_TSC_NCPU(%edi), %esi 125 movl CP_TSC_TYPE(%edi), %eax 126 jmp 4b 127 1289: 129 /* Grab the per-cpu offset and add it to the TSC result */ 130 leal CP_TSC_SYNC_TICK_DELTA(%edi), %esi 131 leal (%esi, %ecx, 8), %ecx 132 addl (%ecx), %eax 133 adcl 0x4(%ecx), %edx 134 jmp 1b 135 SET_SIZE(__cp_tsc_read) 136 137/* 138 * uint_t 139 * __cp_getcpu(uint_t cp_tsc_type) 140 */ 141 ENTRY_NP(__cp_getcpu) 142 /* 143 * If RDTSCP is available, it is a quick way to grab the cpu_id which 144 * is stored in the TSC_AUX MSR by the kernel. 145 */ 146 movl 4(%esp), %eax 147 movl CP_TSC_TYPE(%eax), %eax 148 cmpl $TSC_TSCP, %eax 149 jne 1f 150 rdtscp 151 movl %ecx, %eax 152 ret 1531: 154 mov $GETCPU_GDT_OFFSET, %eax 155 lsl %ax, %eax 156 ret 157 SET_SIZE(__cp_getcpu) 158